diff --git a/.github/scripts/sandbox_vm_provision.sh b/.github/scripts/sandbox_vm_provision.sh index 463faf53..a747a7fa 100755 --- a/.github/scripts/sandbox_vm_provision.sh +++ b/.github/scripts/sandbox_vm_provision.sh @@ -13,7 +13,8 @@ # regardless (§11 item 2). # # Kept dependency-light on purpose: the escape suite imports only stdlib + -# composer.sandbox.* (all stdlib) + pytest, so we install just pytest via uv and +# composer.sandbox.*, whose sole third-party import is annotated_types (three +# files, no dependencies of its own). So we install just pytest + that via uv and # put the repo on PYTHONPATH — no project build, no numpy/psycopg/langchain. We # pass --noconftest so tests/conftest.py (which imports those heavy deps) is not # collected; the suite's fixtures are all in-module. @@ -75,8 +76,10 @@ section "Escape suite against kernel ${KREL}" export PYTHONPATH="${REPO}" # --no-project: don't build AutoProver; --with: ephemeral pytest env. # --noconftest: skip tests/conftest.py's heavy imports (fixtures here are in-module). +# annotated-types is composer.sandbox.config's only non-stdlib import (it carries the +# Ge(0) bound on the Rust-mirrored timeout_s), so importing the package needs it here. uv run --no-project --python 3.12 \ - --with 'pytest>=9.0' --with 'pytest-asyncio>=1.3' \ + --with 'pytest>=9.0' --with 'pytest-asyncio>=1.3' --with 'annotated-types>=0.7' \ pytest --noconftest -v \ --junitxml="${JUNIT}" \ "${REPO}/tests/test_sandbox_escape.py" diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index d8b1b52a..8cefc0e5 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -2,8 +2,8 @@ name: Integration Tests on: schedule: - - cron: '7 5 * * *' - workflow_dispatch: {} # adds a "Run workflow" button in the Actions tab + - cron: "7 5 * * *" + workflow_dispatch: {} # adds a "Run workflow" button in the Actions tab # If a manual run and a scheduled run overlap, cancel the older one # so you don't pay for the expensive job twice. @@ -11,11 +11,17 @@ concurrency: group: scheduled-tests cancel-in-progress: true +permissions: + id-token: write + contents: read + jobs: test: runs-on: ubuntu-latest timeout-minutes: 20 - + environment: prod # Both dev/stg are available as well + env: + CERTORA_LAMBDA_URL: ${{ secrets.LAMBDA_FUNCTION_URL }} steps: - name: Check out code uses: actions/checkout@v4 @@ -31,18 +37,22 @@ jobs: - name: Set up JDK uses: actions/setup-java@v4 with: - distribution: 'temurin' - java-version: '21' + distribution: "temurin" + java-version: "21" - name: sync-deps + # --no-dev skips the `dev` group (and the `apps` group it includes), so this + # job doesn't compile the Rust crates — nothing under `expensive` or `fuzz` + # touches them. Every `uv run` below needs the flag too: a bare `uv run` + # re-syncs with the default groups and would pull `apps` back in. run: | - uv sync --group test --extra prover + uv sync --group test --extra prover --no-dev - name: sync solc run: | uv pip install solc-select - uv run solc-select install 0.8.29 - uv run solc-select use 0.8.29 + uv run --no-dev solc-select install 0.8.29 + uv run --no-dev solc-select use 0.8.29 # certoraRun resolves Certora-convention names like `solc8.29` as # literal executables; solc-select only ships the `solc` shim, so # fetch the static binary under that exact name. @@ -51,14 +61,28 @@ jobs: sudo chmod +x /usr/local/bin/solc8.29 solc8.29 --version + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@e6de054238d6b7531b4efff3b6587d9aade6a06c # v6.2.3 + with: + role-to-assume: ${{ secrets.AWS_OIDC_ROLE_ARN }} + aws-region: us-west-2 + + - name: Basic who-am-i check + run: | + # The stored URL ends in a slash and //who-am-i would not route. + curl -sS --fail-with-body "${CERTORA_LAMBDA_URL%/}/who-am-i" \ + --user "$AWS_ACCESS_KEY_ID":"$AWS_SECRET_ACCESS_KEY" \ + -H "x-amz-security-token: $AWS_SESSION_TOKEN" \ + --aws-sigv4 "aws:amz:us-west-2:lambda" + - name: run autoprover integration tests run: | - uv run pytest -n 3 -m 'expensive' tests + uv run --no-dev pytest -n 3 -m 'expensive' tests env: CERTORAKEY: ${{ secrets.CERTORAKEY }} - name: run template fuzz tests run: | - uv run pytest -n 2 -m 'fuzz' tests + uv run --no-dev pytest -n 2 -m 'fuzz' tests env: HYPOTHESIS_PROFILE: extended diff --git a/.github/workflows/pyright.yml b/.github/workflows/pyright.yml index c77be253..a7f91768 100644 --- a/.github/workflows/pyright.yml +++ b/.github/workflows/pyright.yml @@ -2,9 +2,9 @@ name: pyright-check on: push: - branches: [ main, master ] + branches: [ main, master, dev ] pull_request: - branches: [ main, master ] + branches: [ main, master, dev ] jobs: pyright: @@ -22,10 +22,13 @@ jobs: cache-suffix: pyright - name: sync-deps + # --no-dev skips the `dev` group (and the `apps` group it includes), so this + # job doesn't compile the Rust crates — pyright can't see into a compiled + # extension module anyway. run: | - uv sync --group ci + uv sync --group ci --no-dev uv pip install sentence_transformers --no-deps - name: Run pyright run: | - uv run pyright composer/ analyzer sanity_analyzer certora_autosetup + uv run --no-dev pyright composer/ analyzer sanity_analyzer certora_autosetup diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index c0987e86..f4846e3f 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -2,9 +2,9 @@ name: pytest-check on: push: - branches: [ main, master ] + branches: [ main, master, dev ] pull_request: - branches: [ main, master ] + branches: [ main, master, dev ] jobs: pytest: @@ -20,7 +20,29 @@ jobs: enable-cache: true cache-dependency-glob: "uv.lock" cache-suffix: pytest - + + # `dev` is one of uv's default groups and includes `apps`, so the sync below + # builds the `rust/` wheels — tests/test_rustapp.py and the launcher suites + # skip themselves without them. uv builds those crates concurrently, and + # rustup's install path is not safe against concurrent invocations: whichever + # maturin -> cargo call wins clears $RUSTUP_HOME/downloads and the other one + # dies renaming its half-downloaded component. So install the toolchain + # serially, first. No argument — rustup takes the channel, profile and + # components from rust-toolchain.toml. + - name: Install the pinned Rust toolchain + run: rustup toolchain install --no-self-update + + - name: Cache the Rust build + uses: actions/cache@v4 + with: + path: | + ~/.cargo/registry/index + ~/.cargo/registry/cache + ~/.cargo/git/db + rust/target + key: cargo-pytest-${{ hashFiles('rust/Cargo.lock') }} + restore-keys: cargo-pytest- + - name: sync-deps run: | uv sync --group test --extra prover diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index dcf96eb4..9e06bbca 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -115,7 +115,7 @@ five steps and never inspects anything backend-specific: `run_pipeline` also takes an `ecosystem: Ecosystem[App, Main, Unit]` ([composer/pipeline/ecosystem.py](composer/pipeline/ecosystem.py)) — the *front-half* plug point, orthogonal to the backend. It pairs a **language** (how the target's source is read — fs-exclusion -pattern, code-explorer prompt) with a **chain** and supplies the domain-specific pieces the shared +pattern) with a **chain** (including the code-explorer prompt) and supplies the domain-specific pieces the shared steps need: the analyzed-model type (`App`), the analysis/property prompts, model validation, how to locate the target `Main`, and `units(main) -> list[Unit]` (the per-unit split the extraction and formalization phases iterate). `EVM` binds `(SourceApplication, ContractInstance, diff --git a/CLAUDE.md b/CLAUDE.md index f45661b9..e06cd465 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -29,6 +29,12 @@ loose). Both flags matter: every non-default group the two commands above need, `ci` (pyright) included, or the sync that fixes one of them breaks the other: `uv sync --group test --group ci --group ragbuild --extra cpu --extra certora-cli`. +- **The Rust toolchain.** That sync's default `dev` group builds the maturin crates under + `rust/`, so it needs cargo. If the toolchain `rust-toolchain.toml` pins is not installed + yet, install it first and on its own — `rustup toolchain install --no-self-update`, no + argument, so rustup reads the channel, profile and components from that file. uv builds + those crates concurrently and rustup's on-demand install is not concurrency-safe, so + letting the build trigger it races. ## Python diff --git a/README.md b/README.md index 39fc582b..fc1d4010 100644 --- a/README.md +++ b/README.md @@ -40,11 +40,28 @@ The rest of this document covers the **host `uv` flow** for development. You need everything from the [AIComposer infrastructure setup](AICOMPOSER_INFRA.md): - Python 3.12+, `uv`, Docker with compose +- `rustup` — a default `uv sync` builds the `rust/` crates (see below) - `ANTHROPIC_API_KEY` in your environment - PostgreSQL databases running (see below) - RAG database populated - Solidity compiler(s) on `$PATH` (naming convention `solcX.Y`, e.g. `solc8.29`) +### Rust toolchain + +A `uv sync`, or any `uv run`, which revalidates path dependencies, builds the maturin crates under `rust/`. Without a +toolchain the sync fails outright rather than skipping. Install [rustup](https://rustup.rs) (a distro `cargo` package is +not enough: `rust-toolchain.toml` pins the toolchain and rustup is what reads it), then once, from the repo root: + +```bash +rustup toolchain install --no-self-update +``` + +Re-run that after any bump to `rust-toolchain.toml`'s `channel` — rustup would otherwise install the new toolchain from +inside the build, where the concurrent cargo calls a sync makes race each other and one dies mid-download. + +Not working on the Rust side? `uv sync --no-dev` skips the crates entirely — the Rust tests then skip themselves instead +of failing, and the CI pyright job runs this way. + ### Certora Prover **Cloud mode (the default):** Two credentials are needed. `CERTORAKEY` (set in your environment) authenticates `certoraRun` when it submits prover jobs. The job *results* are fetched with the Certora cloud credentials produced by `certora-cloud login` — install the public CLI (`uv tool install certora-cloud`) and run it once (`certora-cloud login`); it writes `~/.certora/credentials.json`, which is read automatically. Exporting `CERTORA_USER`/`CERTORA_TOKEN`/`CERTORA_REFRESH_TOKEN` instead is an optional override. diff --git a/budget_integration.md b/budget_integration.md new file mode 100644 index 00000000..8850fdd1 --- /dev/null +++ b/budget_integration.md @@ -0,0 +1,240 @@ +# Run budgets: integration guide + +This document is for the frontend and cloud-orchestration owners. It covers how to +launch a budgeted AutoProve run, what a budget does to the run's outputs when it +trips, and what your side needs to handle: the new report fields, the on-disk +artifacts, and the operational caveats of the metering. + +A *run budget* is a USD ceiling on LLM spend, enforced inside the pipeline while it +runs. When spend approaches a limit, in-flight agents are told to wrap up and publish +their best partial result; when spend exceeds the limit, they are terminated +cooperatively. A component whose formalization was cut short this way is **curtailed**: +it is not a delivery, its outputs are quarantined on disk, and it is reported in a +dedicated appendix of `report.json` rather than in the main property/rule tables. + +## 1. The budget file and the `--budget` flag + +Both pipeline entry points (prover and foundry) accept: + +``` +--budget path/to/budget.json +``` + +Omitting the flag runs unbudgeted — nothing about an unbudgeted run's behavior or +outputs changes. + +The file is JSON natively; a `.yaml`/`.yml` file also works when PyYAML happens to be +installed (it is *not* a declared dependency — cloud environments should stick to +JSON). Schema: + +```json +{ + "total": 25.0, + "caps": { + "formalization": 15.0, + "property_extraction": 5.0 + } +} +``` + +- `total` (required, USD, > 0) — the run **pool**, the real bound on overall spend. +- `caps` (optional, USD each, >= 0) — per-phase ceilings, any subset of the five + phase names below. A phase without an explicit cap defaults to `total`, i.e. it is + bounded by the pool alone. + +Caps are **ceilings, not allotments**: they bound how much a single phase may hog and +need not sum to `total`. Whatever a phase doesn't spend simply stays in the pool for +later phases — rollover is automatic. Cost accrues to the active phase cap *and* the +pool simultaneously; enforcement trips on whichever is tighter. + +The five phase names (the keys of `PhaseBudget` in `composer/pipeline/ptypes.py`): + +| Phase | Covers | +|---|---| +| `system_analysis` | component analysis of the contract system | +| `system_preparation` | harness construction | +| `formalization_preparation` | prover pre-formalization fan-out: structural invariants, custom summaries, protocol-specific setup | +| `property_extraction` | per-component property inference | +| `formalization` | per-component CVL / test authoring (usually the dominant cost) | + +Work outside any named phase — notably the report/grouping step at the end — accrues +to (and feels pressure from) the pool directly. + +Validation is strict and fail-fast: unknown top-level keys, unknown phase names, +`total <= 0`, or a negative cap all reject the file with a `ValueError` **before any +services spin up**, so a malformed budget fails the run immediately at startup rather +than mid-pipeline. A cap of `0.0` is legal: that phase starts already inside its +wrap-up window (useful for forcing curtailment in tests). + +The parsed budget is echoed into the run's data log as a `budget` record +(`{"total": ..., "caps": {...}}`), so run records show what the run was launched with. + +### Enforcement semantics (what "trips" means) + +Two thresholds, per counter (each phase cap, and the pool): + +- **Wrap-up window** — at **80%** of a counter, the running agent gets a one-time + injected warning telling it to finish in an orderly fashion, and its validation + gates are lifted so it can publish a best-effort partial result. The pipeline also + stops launching new work that would immediately be told to pack it in (e.g. further + property-extraction rounds). +- **Hard stop** — past **100%**, the agent is terminated cooperatively between turns. + +Both checks run *between* agent turns; see the caveats in §4 for what "cooperatively" +implies about overshoot. + +## 2. Curtailment in `report.json` + +`report.json` (written to `certora/ap_report/report.json`) keeps `schema_version: +"3.0"` — curtailment adds fields, it does not break existing ones. + +A curtailed component contributes **nothing** to the main tables: it is excluded from +`properties`, `rules`, `groups`, `skipped`, and `prover_links`. Whatever it published +was accepted with the validation gates lifted, so neither the encoding nor any +verification result it saw along the way is reliable. Instead it appears in the +appendix list: + +```jsonc +"curtailed_components": [ + { + "component": "Withdrawals", + // Project-relative path of the quarantined partial encoding (see §3); + // null when the run stopped before anything was published. + "artifact": "certora/specs/Withdrawals.spec.unverified", + // Last verification-run link the partial carried, if any. Context only — + // its outcome predates the final content and proves nothing about it. + "run_link": "https://prover.certora.com/output/...", + // Optional account of the termination (hard-stop message, or the + // author's own words). + "detail": "Token cost budget exhausted; the agent was cooperatively terminated.", + // The component's inferred properties, partitioned by disposition: + "drafted": [ + // claims the author made before being cut off — encoded but UNVERIFIED + { "title": "solvency_preserved", "sort": "safety_property", + "description": "...", "units": ["solvencyPreserved"] } + ], + "skipped": [ + // properties the author explicitly declined (usually citing the budget) + { "title": "reentrancy_safe", "sort": "attack_vector", + "description": "...", "reason": "insufficient budget remaining" } + ], + "unattempted": [ + // properties the author never reached + { "title": "fee_bounded", "sort": "invariant", "description": "..." } + ] + } +] +``` + +Field notes for the FE: + +- Every property record carries the standard formulation fields `title` (unique + snake_case id), `sort` (`attack_vector` | `safety_property` | `invariant`), and + `description`. +- `drafted[].units` are the rule/test names the author *declared* at publish — an + unchecked claim, never validated against a judge or a verification run. Do not + render them as results. +- All three lists can be empty; a component cut off before publishing anything has + `artifact: null` and typically everything under `unattempted`. + +`coverage` gains: + +- `curtailed_component_count` — the appendix length; +- a `warnings` entry when it is nonzero: `"N component(s) were cut short by the run + budget; their properties are listed in the curtailed appendix, not the groups."` + +Rendering guidance: treat `curtailed_components` as a first-class "cut short by +budget" section, visually distinct from both delivered results and `gave_up_components` +(a give-up is the author's considered infeasibility judgment; curtailment is the run +running out of money). + +One more report-level behavior: when *zero* properties were formalized run-wide +(everything gave up or was curtailed), the report is still written, but the LLM +grouping step is skipped — expect `groups: []` with a populated appendix. + +## 3. Disk semantics + +**Quarantine.** A curtailed component's partial encoding is persisted for inspection +under a poisoned name: the normal artifact filename plus the suffix `.unverified` +(e.g. `Withdrawals.spec.unverified`, `Counter.t.sol.unverified`), in the same +directory as real deliverables. Only the artifact text is written — **no** `.conf`, +no commentary, no property-map JSON — so nothing runnable or machine-readable points +at content that never passed the validation gates. The suffix also keeps forge from +compiling a quarantined `.t.sol`. (The analysis-phase +`properties/{stem}.properties.json` *is* present — it is written before formalization +begins, for every attempted component regardless of outcome.) + +**Caching.** Curtailed results are never cached. A re-run of the same project redoes +the curtailed components from scratch (and, given budget, delivers them properly); +already-delivered components come from cache as usual. + +**`components_to_prover_runs.json`** (`.certora_internal/autoProve/`): deliveries +only. Curtailed components never get an entry — a partial's last run link says +nothing about its final content. + +**Exit code.** The console entry points exit `1` only when *every* attempted +component ended without a deliverable (`all_failed`: gave up, crashed, or curtailed). +A run where some components delivered and some were curtailed exits `0` — +orchestration must read `coverage.curtailed_component_count` (or the appendix) to +detect partial curtailment; the exit code will not tell you. Each curtailed component +also appears in the result's failure list, printed at the end of a console run as +`NAME: BUDGET: formalization cut short (unvalidated partial kept at PATH | nothing +published)`. + +## 4. Operational caveats + +- **Enforcement is cooperative, not preemptive.** Budget checks run between agent + turns; a turn already in flight completes and its cost lands after the fact. With + several components formalizing in parallel, each can have a turn in flight when the + pool trips. Treat `total` as a target with margin — worst case overshoot is roughly + one expensive turn per concurrently running agent — not as a billing guarantee. +- **The 80% threshold is a global constant** (`BUDGET_PRESSURE_THRESHOLD`, + `composer/diagnostics/budget.py`), not a per-run knob. Consequence: the wrap-up + window is only 20% of the cap. For small caps that can be about one worst-case turn + of headroom, so an agent may hard-stop *mid-wrap-up* — orchestration and the FE + must tolerate curtailed components with `artifact: null` even though the budget + "asked nicely" first. +- **A model missing from the pricing table accrues $0.** The meter prices each call + via `composer/llm/pricing.py`; an unpriced model spends invisibly and the budget + never trips on it. Silent in production — keep the pricing table in sync with the + model roster. +- **Warm caches spend ~$0.** Components served from the generation cache skip + authoring entirely, so a budget calibrated for a cold run will never trip on a + re-run against the same cache namespace. Conversely (per §3) curtailed work is + never cached, so re-running a curtailed run re-spends on exactly those components. +- **Autosetup subprocess spend is invisible.** LLM calls made inside the autosetup + subprocess do not flow through the meter and count toward no cap. +- **Sub-agents accrue to their parent's phase.** Judges, researchers, and other + spawned helpers spend from whatever named scope their root agent runs under; there + is no separate accounting knob for them. +- **The report step draws on the pool only.** It has no named cap; a run that arrives + at the report phase with an exhausted pool will feel pressure there too. + +## 5. Calibration and cost telemetry + +To size budgets from real runs, use `composer/scripts/budget_math.py`: + +``` +uv run composer/scripts/budget_math.py [--uid U] + [--cap-fraction 0.75] [--wrapup-turns 4] [--emit-matrix DIR] +``` + +It replays a run trail (an `ap-trail export` dump, or a bare run id fetched live from +the store) and reconstructs, per LLM call, exactly what the cost meter would have +accrued — at both cache-write TTLs, since the trail doesn't record which TTL a +conversation used. Calibrate on the 1h bound (the authors run with the long cache). +Sub-agent threads fold into their root thread, matching how budget scopes accrue. + +`--emit-matrix DIR` writes a ready-made live-test budget matrix (a control budget +plus budgets that trip the formalization cap, the preparation cap, and the shared +pool) with a `manifest.md` of expected outcomes per test. + +Phase attribution rides on **cost-center telemetry**: every logged thread records the +named budget scope it ran under (`ThreadMeta.cost_center`), stamped unconditionally — +budget or no budget — so phase attribution works on unbudgeted runs too. `null` means +pool-level or pre-pipeline work; trails recorded before cost-center tracking existed +lack the field and classify as unattributed. + +Note the distinction from the existing token accounting: `ap_report/job_info.json` +and `token_usage.json` report token *counts* by model and by workflow phase; the +budget meters *USD* via the pricing table. They are related but not the same numbers. diff --git a/certora_autosetup/autosetup/autosetup.py b/certora_autosetup/autosetup/autosetup.py index 0337b4b0..2b3a837e 100644 --- a/certora_autosetup/autosetup/autosetup.py +++ b/certora_autosetup/autosetup/autosetup.py @@ -57,6 +57,11 @@ COMPONENT = "Autosetup" +# Per-job prover timeout (seconds) for the Collect Difficulties run. It skips the SMT solve, so this +# is a safety net for the build/optimize pipeline, not the solver — a heavy method must reach the +# postOptimize surviving-graph dump the detector consumes, so it is generous (20m). +DIFFICULTY_RUN_GLOBAL_TIMEOUT = 1200 + def _reconcile_evm_version_keys( config: Dict[str, Any], @@ -101,6 +106,13 @@ class Autosetup: "callTraceHardFail": "on", } + DIFFICULTY_RUN_PROVER_ARGS = { + "skipFormulaChecking": "", + "callTraceHardFail": "off", + # Emit the postOptimize SurvivingCallGraph the summarization detector reads (off by default). + "dumpSurvivingCallGraph": "true", + } + def __init__( self, config: AutosetupConfig, @@ -140,6 +152,7 @@ def __init__( self.import_patcher_applied: bool = False self._sanity_advanced_analysis: Dict[str, Dict[str, SanityFailureResult]] = {} self._test_run_specs: List[ProverJobSpec[Any]] = [] + self._difficulty_run_specs: List[ProverJobSpec[Any]] = [] self.bytes_mappings: List[tuple[ContractHandle, List[str]]] = [] # Set during run() self.main_contract_handle: Optional[ContractHandle] = None @@ -345,6 +358,7 @@ def run(self, main_contract_handle: ContractHandle, skip_warmup: bool = False) - sanity_analysis=dict(self._sanity_advanced_analysis), bytes_mappings=list(self.bytes_mappings), test_run_specs=list(self._test_run_specs), + difficulty_run_specs=list(self._difficulty_run_specs), build_system_config_dict=self.get_build_system_config_dict_with_updates(), orchestration_timestamp=self.config.orchestration_timestamp, # Every LLM call this process made (summaries, proxy detection, @@ -418,7 +432,12 @@ def is_file_in_scope(self, file_path): # Get appropriate manager class and create instance ManagerClass = BuildSystemDetector.get_manager_class(detected) - manager: BuildSystemManager = ManagerClass(self.build_config_dir, scope) # type: ignore + # The manager is anchored on the build config dir, but remapping contexts and the + # hoisted-package walk belong to the directory certoraRun runs from — the two differ + # exactly when the build config lives in a monorepo sub-project. + manager: BuildSystemManager = ManagerClass( # type: ignore + self.build_config_dir, scope, run_root=run_root + ) # Auto-detect and parse config (polymorphic - returns FoundryConfig or HardhatConfig) self.log(f"Auto-detecting {detected.value} configuration...") @@ -853,26 +872,47 @@ async def prepare_contract_warmup() -> Optional[ProverJobSpec[Any]]: contract_advanced = await autosetup.set_sanity_options(enhanced_config.path, contract_name) autosetup._sanity_advanced_analysis.update(contract_advanced) - # Create test run config BEFORE building the warmup config, because the test flags - # need actual sanity rules to exercise. Lands in .certora_internal/confs/ so the - # user-facing certora/confs/ stays a single-file directory. + # Both runs land in .certora_internal/confs/ so the user-facing certora/confs/ stays a + # single-file directory. Built BEFORE the warmup config because they need actual sanity + # rules to exercise. internal_confs_dir = autosetup.config.project_root / DIR_INTERNAL_CONFS - test_config = autosetup.config_manager.create_copy_with_prover_args( + + difficulty_config = autosetup.config_manager.create_copy_with_prover_args( enhanced_config.path, - autosetup.TEST_RUN_PROVER_ARGS, - "_test_run", + autosetup.DIFFICULTY_RUN_PROVER_ARGS, + "_difficulty", target_dir=internal_confs_dir, ) - # The test run exercises the generated sanity rule; rule_sanity's own - # checks would duplicate it, so they are turned off for this invocation - # only (via extra_args, leaving the conf's rule_sanity untouched). - autosetup._test_run_specs.append(ProverJobSpec( - config_file=test_config, + autosetup.config_manager.update_config_with_properties( + difficulty_config.path, {"global_timeout": DIFFICULTY_RUN_GLOBAL_TIMEOUT} + ) + autosetup.config_manager.update_config_with_prover_args( + difficulty_config.path, remove_args=["destructiveOptimizations"] + ) + autosetup._difficulty_run_specs.append(ProverJobSpec( + config_file=difficulty_config, contract_name=contract_name, - phase=f"Sanity Test Run - {contract_name}", + phase=f"Collect Difficulties - {contract_name}", extra_args=[*autosetup.config.extra_args, "--rule_sanity", "none"], )) + if not autosetup.config.skip_test_run: + test_config = autosetup.config_manager.create_copy_with_prover_args( + enhanced_config.path, + autosetup.TEST_RUN_PROVER_ARGS, + "_test_run", + target_dir=internal_confs_dir, + ) + # The test run exercises the generated sanity rule; rule_sanity's own + # checks would duplicate it, so they are turned off for this invocation + # only (via extra_args, leaving the conf's rule_sanity untouched). + autosetup._test_run_specs.append(ProverJobSpec( + config_file=test_config, + contract_name=contract_name, + phase=f"Sanity Test Run - {contract_name}", + extra_args=[*autosetup.config.extra_args, "--rule_sanity", "none"], + )) + if skip_warmup: return None diff --git a/certora_autosetup/autosetup/cli.py b/certora_autosetup/autosetup/cli.py index a95ee140..ffd21091 100644 --- a/certora_autosetup/autosetup/cli.py +++ b/certora_autosetup/autosetup/cli.py @@ -27,8 +27,9 @@ FILE_AUTOSETUP_RESULT, FILE_LLM_USAGE, FILE_PROVER_USAGE, + FILE_SUMMARIZATION_CANDIDATES, ) -from certora_autosetup.utils.contract_utils import auto_detect_contracts, deduplicate_contract_handles, parse_contract_files, resolve_contract_handles, split_contract_spec +from certora_autosetup.utils.contract_utils import auto_detect_contracts, deduplicate_contract_handles, parse_contract_files, resolve_contract_handles, split_contract_spec, with_contract_handle from certora_autosetup.utils.project_dir import find_build_config_dir from certora_autosetup.utils.enhanced_config_manager import ConfigManager from certora_autosetup.utils.llm_util import LlmUsageReport, ledger_reset @@ -82,10 +83,31 @@ def main(): contract_handles = deduplicate_contract_handles(contract_handles) - # Parse main contract - main_handles = parse_contract_files([args.main_contract]) + # Parse main contract. It goes through the same artifact-backed name resolution as + # --contract-files-and-name, so a bare `path.sol` spec gets the contract the file really + # declares instead of the filename stem. + main_handles = resolve_contract_handles( + parse_contract_files([args.main_contract]), build_config_dir, profile=args.profile, + requested_build_system=args.build_system, handles_relative_to=cwd, + ) main_contract_handle = main_handles[0] + # A contract the caller named is in scope by definition, but two upstream steps can drop + # it: auto-detection skips every file under a dependency directory (node_modules/, lib/, + # dependencies/, ...), which is where per-address verification bundles and vendored + # sub-projects keep real, deployed code, and deduplicate_contract_handles prefers the + # shortest path when two files declare the same contract name. Either way the run fails + # much later, from setup_prover, as "is not among the compiled contracts in the prover + # scene" — a compilation message for what is really a scoping decision. + if main_contract_handle not in contract_handles: + logger.log( + f"Main contract {main_contract_handle.contract_name}@" + f"{main_contract_handle.source_file} was not among the auto-detected contracts " + f"— adding it to the scene", + "INFO", "Autosetup", + ) + contract_handles = with_contract_handle(contract_handles, main_contract_handle) + # TODO: a bare `path.sol` spec drops only the contract whose name matches the file # stem. Expand to "drop every concrete contract in the file" for symmetry with # auto-detect's emit-all default. Mirror the same expansion for include specs @@ -203,6 +225,7 @@ def main(): no_strip_contracts=args.no_strip_contracts, include_foundry_packages=not args.exclude_foundry_packages, run_source=args.run_source, + skip_test_run=args.skip_test_run, ), setup_prover=setup_prover, prover_runner=prover_runner, @@ -237,8 +260,10 @@ def main(): Path(args.composer_setup).write_text(json.dumps(result.composer_output, indent=2)) print(f"Composer output written to: {args.composer_setup}") - # Submit test run jobs and generate reports via ConfRunner - if result.test_run_specs: + # Submit the deferred prover jobs (Collect Difficulties + optional Sanity Test Run) and generate + # reports via ConfRunner. + deferred_specs = result.difficulty_run_specs + result.test_run_specs + if deferred_specs: from certora_autosetup.reporting.json_reporter import JsonReporter from certora_autosetup.setup.setup_completeness_checker import SetupCompletenessChecker, SetupCompletenessReport from certora_autosetup.conf_runner import ConfRunner, ConfRunnerConfig @@ -264,14 +289,41 @@ def main(): project_root=project_root, certora_dir=certora_dir, ) - conf_runner.run_confs( + test_results, _ = conf_runner.run_confs( config_files=[], - test_run_specs=result.test_run_specs, + test_run_specs=deferred_specs, sanity_analysis=result.sanity_analysis, bytes_mappings=result.bytes_mappings, llm_usage=ledger_rows, ) + # Summarization-target detector: from the completed Collect Difficulties run, rank the + # prover-hostile functions worth summarizing (and, for a curated match, how). Written for composer + # to ingest, alongside the prover-usage / llm-usage artifacts below. Best-effort — a detector + # failure never fails autosetup. + try: + from summarization_detector.detect import detect + from summarization_detector.sources import fetch_surviving_graphs + + difficulty_url = next( + (r.job_url for r in test_results + if r.job_url and r.job_handle.phase.startswith("Collect Difficulties")), + None, + ) + if difficulty_url and result.asts_path: + report = detect( + difficulty_url, + ast_path=result.asts_path, + cut=main_contract_handle.contract_name, + surviving_graphs=fetch_surviving_graphs(difficulty_url), + sources_root=project_root, + ) + out = reports_dir / FILE_SUMMARIZATION_CANDIDATES + out.write_text(json.dumps(report.to_dict(), indent=2)) + print(f"Summarization candidates written to: {out}") + except Exception as e: # noqa: BLE001 — best-effort side artifact + print(f"[autosetup] summarization detector skipped: {e}", file=sys.stderr) + # Persist this run's prover-reported runtime (only the jobs actually executed; # cache hits are excluded by the runner ledger) for composer to ingest. Mirrors # llm_usage.json. Written last, after conf_runner's prover runs have completed. diff --git a/certora_autosetup/autosetup/cli_args.py b/certora_autosetup/autosetup/cli_args.py index c8a35327..88ec782e 100644 --- a/certora_autosetup/autosetup/cli_args.py +++ b/certora_autosetup/autosetup/cli_args.py @@ -130,6 +130,12 @@ def create_parser(): help='Skip cache warmup phase' ) + parser.add_argument( + '--skip-test-run', + action='store_true', + help='Skip the sanity Test Run (and its setup-completeness / comprehensive reports).' + ) + # Generator options (all enabled by default) parser.add_argument( '--disable-extcall', diff --git a/certora_autosetup/autosetup/types.py b/certora_autosetup/autosetup/types.py index 81e25eca..2c79dcb8 100644 --- a/certora_autosetup/autosetup/types.py +++ b/certora_autosetup/autosetup/types.py @@ -32,9 +32,6 @@ class AutosetupConfig: # Feature flags skip_sanity_setup: bool = False - # When True, skip the AIComposer-backed sanity coverage analysis (the per-method - # coverage rerun jobs + sanity_analyzer vacuity analysis). Loop-iter and hashing-bound - # detection still run. Set by PreAudit, which does not consume the advanced analysis. skip_sanity_coverage_analysis: bool = False skip_hashing_bound_detection: int | None = None min_loop_iter: int = 3 @@ -42,6 +39,7 @@ class AutosetupConfig: skip_call_resolution: bool = False skip_proxy_detection: bool = False skip_harnessing: bool = False + skip_test_run: bool = False no_strip_contracts: bool = False keep_intermediate_typechecker_files: bool = False dummy_erc20: int | None = None @@ -80,6 +78,9 @@ class AutosetupResult: # Deferred sanity test run jobs (created during warmup, submitted by ConfRunner) test_run_specs: list = field(default_factory=list) # list[ProverJobSpec] + # Deferred Collect Difficulties jobs (skip the SMT solve; feed the summarization detector) + difficulty_run_specs: list = field(default_factory=list) # list[ProverJobSpec] + # Build system info (needed by conf_runner for merging into checker confs) build_system_config_dict: dict[str, Any] = field(default_factory=dict) diff --git a/certora_autosetup/build_systems/foundry.py b/certora_autosetup/build_systems/foundry.py index 191448a8..f49c844d 100644 --- a/certora_autosetup/build_systems/foundry.py +++ b/certora_autosetup/build_systems/foundry.py @@ -124,15 +124,16 @@ class FoundryManager(BuildSystemManager): error handling, and integrated compilation management. """ - def __init__(self, project_root: Path, scope): + def __init__(self, project_root: Path, scope, run_root: Optional[Path] = None): """ Initialize foundry manager. Args: - project_root: Root directory of the project + project_root: Directory the foundry.toml is anchored on scope: Centralized scope for consistent filtering + run_root: Directory certoraRun is invoked from (defaults to project_root) """ - super().__init__(project_root, scope, "FoundryManager") + super().__init__(project_root, scope, "FoundryManager", run_root=run_root) def get_config_filenames(self) -> List[str]: """Return list of config filenames to search for.""" @@ -182,7 +183,10 @@ def parse_config(self, config_file: Path, profile: str | None = None) -> Foundry # Build the packages list from forge remappings + foundry.toml + remappings.txt # + package.json for the resolved profile. packages = build_packages_from_remapping_sources( - base_dir=config_file.parent, log_fn=self.log, profile=profile + base_dir=config_file.parent, + log_fn=self.log, + profile=profile, + run_root=self.run_root, ) if packages: config.packages = packages @@ -390,6 +394,15 @@ def get_build_command(self, profile: Optional[str] = None) -> str: return f"FOUNDRY_PROFILE={profile} forge build" return "forge build" + @staticmethod + def holds_artifacts(artifacts_dir: Path) -> bool: + """Foundry writes one `.sol/` directory per compiled source file.""" + if not artifacts_dir.is_dir(): + return False + return any( + child.is_dir() and child.name.endswith(".sol") for child in artifacts_dir.iterdir() + ) + def filter_artifacts(self, artifacts_dir: Path) -> List[Path]: """ Filter Foundry artifacts - all .json files except those in build-info/ directories. diff --git a/certora_autosetup/build_systems/hardhat.py b/certora_autosetup/build_systems/hardhat.py index 5a7d76d1..1585a56c 100644 --- a/certora_autosetup/build_systems/hardhat.py +++ b/certora_autosetup/build_systems/hardhat.py @@ -76,15 +76,18 @@ class HardhatManager(BuildSystemManager): Parallel to FoundryManager but adapted for Hardhat's JavaScript/TypeScript ecosystem. """ - def __init__(self, project_root: Path, scope): + def __init__(self, project_root: Path, scope, run_root: Optional[Path] = None): """ Initialize Hardhat manager. Args: - project_root: Root directory of the project + project_root: Directory the hardhat config is anchored on scope: Centralized scope for consistent filtering + run_root: Directory certoraRun is invoked from (defaults to project_root). + Accepted for interface parity — autosetup constructs every manager class + through the same call. """ - super().__init__(project_root, scope, "HardhatManager") + super().__init__(project_root, scope, "HardhatManager", run_root=run_root) def get_config_filenames(self) -> List[str]: """Return list of config filenames to search for.""" @@ -381,6 +384,16 @@ def get_build_command(self, profile: Optional[str] = None) -> str: """Return Hardhat build command.""" return "npx hardhat compile" + @staticmethod + def holds_artifacts(artifacts_dir: Path) -> bool: + """Hardhat mirrors the sources tree under the artifacts dir and writes `build-info/` + beside it; the mirror is named after `paths.sources`, so `build-info/` is the part + that is there whatever the project calls its sources.""" + if not artifacts_dir.is_dir(): + return False + return (any((artifacts_dir / "contracts").rglob("*.json")) + or any((artifacts_dir / "build-info").glob("*.json"))) + def filter_artifacts(self, artifacts_dir: Path) -> List[Path]: """ Filter Hardhat artifacts - only contracts/, exclude .dbg.json and build-info/. diff --git a/certora_autosetup/build_systems/manager.py b/certora_autosetup/build_systems/manager.py index 0bd30555..63724c4f 100644 --- a/certora_autosetup/build_systems/manager.py +++ b/certora_autosetup/build_systems/manager.py @@ -11,7 +11,7 @@ import sys from abc import ABC, abstractmethod from pathlib import Path -from typing import Callable, List, Set +from typing import Callable, List, Optional, Set from certora_autosetup.build_systems.base import BuildSystemConfig @@ -25,16 +25,22 @@ class BuildSystemManager(ABC): build-system-specific parsing and command generation. """ - def __init__(self, project_root: Path, scope, component_name: str): + def __init__(self, project_root: Path, scope, component_name: str, run_root: Optional[Path] = None): """ Initialize build system manager. Args: - project_root: Root directory of the project + project_root: Directory the build config is anchored on — where config discovery + starts and artifacts are read from. In a monorepo this is the sub-project that + owns the main contract, not the run root. scope: Centralized scope for consistent filtering component_name: Name for logging (e.g. "FoundryManager", "HardhatManager") + run_root: Directory certoraRun is invoked from. Remapping contexts are expressed + against it and the hoisted-package walk is bounded by it. Defaults to + project_root, which is correct whenever the build config sits at the run root. """ self.project_root = project_root + self.run_root = run_root or project_root self.scope = scope self.component = component_name @@ -101,6 +107,27 @@ def get_build_command(self, profile: str | None = None) -> str: """ pass + @staticmethod + @abstractmethod + def holds_artifacts(artifacts_dir: Path) -> bool: + """ + Whether *artifacts_dir* holds output written by this build system. + + Recognises the build system's own layout inside the directory, so a directory that + merely exists under the expected name does not pass for a built project. That + happens for real: a project whose configured output dir is nested (Foundry's + ``out = "out/foundry"``) has a bare ``out/`` holding only subdirectories, and a + project that shipped a second build config often has an empty artifact dir left by + the tool that no longer runs. + + Args: + artifacts_dir: Directory to inspect; need not exist + + Returns: + True if the directory holds this build system's artifacts + """ + pass + @abstractmethod def filter_artifacts(self, artifacts_dir: Path) -> List[Path]: """ diff --git a/certora_autosetup/build_systems/truffle.py b/certora_autosetup/build_systems/truffle.py index 282eaaa9..a96db48d 100644 --- a/certora_autosetup/build_systems/truffle.py +++ b/certora_autosetup/build_systems/truffle.py @@ -75,15 +75,16 @@ def get_artifact_directory(self) -> str: class TruffleManager(BuildSystemManager): """Truffle project manager: config parsing and artifact discovery.""" - def __init__(self, project_root: Path, scope): + def __init__(self, project_root: Path, scope, run_root: Optional[Path] = None): """ Initialize Truffle manager. Args: - project_root: Root directory of the project + project_root: Directory the truffle config is anchored on scope: Centralized scope for consistent filtering + run_root: Directory certoraRun is invoked from (defaults to project_root) """ - super().__init__(project_root, scope, "TruffleManager") + super().__init__(project_root, scope, "TruffleManager", run_root=run_root) def get_config_filenames(self) -> List[str]: """Return list of config filenames to search for.""" @@ -97,6 +98,11 @@ def get_build_command(self, profile: str | None = None) -> str: """Return the build command for this build system.""" return "npx truffle compile" + @staticmethod + def holds_artifacts(artifacts_dir: Path) -> bool: + """Truffle writes one flat `.json` per contract into its build dir.""" + return artifacts_dir.is_dir() and any(artifacts_dir.glob("*.json")) + def filter_artifacts(self, artifacts_dir: Path) -> List[Path]: """Return Truffle's artifact JSONs — one flat `.json` per contract.""" return self._walk_and_filter_artifacts( @@ -126,7 +132,9 @@ def parse_config(self, config_file: Path, profile: str | None = None) -> Truffle # Independent of whether the config itself evaluated: the packages list comes from # package.json/node_modules, which is what Truffle's own resolver uses. - packages = build_packages_from_remapping_sources(base_dir=config_file.parent, log_fn=self.log) + packages = build_packages_from_remapping_sources( + base_dir=config_file.parent, log_fn=self.log, run_root=self.run_root + ) if packages: config.packages = packages diff --git a/certora_autosetup/fixconf.py b/certora_autosetup/fixconf.py index ac026fcf..5a3c281e 100644 --- a/certora_autosetup/fixconf.py +++ b/certora_autosetup/fixconf.py @@ -24,7 +24,10 @@ ) from certora_autosetup.parsers.build_system_detector import BuildSystem, BuildSystemDetector from certora_autosetup.setup.solidity_import_patch import apply_patch, create_patch, revert_patch -from certora_autosetup.utils.compilation_workarounds import CompilationWorkaroundManager +from certora_autosetup.utils.compilation_workarounds import ( + CompilationWorkaroundManager, + UnsatisfiableSolcPinError, +) from certora_autosetup.utils.constants import DEFAULT_SOLC_VERSION from certora_autosetup.utils.contract_utils import parse_contract_files from certora_autosetup.utils.logger import logger @@ -172,9 +175,16 @@ def fix_conf( workaround_mgr = CompilationWorkaroundManager( project_root, DEFAULT_SOLC_VERSION, verbose, solc_convention=solc_convention ) - success, output, updated_config_dict = workaround_mgr.run_compilation_with_workarounds( - cmd, working_conf, working_conf_dict, contracts, updated_config_dict - ) + try: + success, output, updated_config_dict = workaround_mgr.run_compilation_with_workarounds( + cmd, working_conf, working_conf_dict, contracts, updated_config_dict + ) + except UnsatisfiableSolcPinError as e: + # The conf pins a compiler this machine cannot provide. Report it and keep + # going: the fixes already written to the working conf are still worth + # handing back. + logger.log(str(e), "ERROR", "fixconf") + success, output = False, str(e) # Import patcher fallback import_patcher_applied = False diff --git a/certora_autosetup/parsers/base.py b/certora_autosetup/parsers/base.py index 8a51d0dc..cccd098e 100644 --- a/certora_autosetup/parsers/base.py +++ b/certora_autosetup/parsers/base.py @@ -80,24 +80,59 @@ def extract_logic_contracts_impl(self, artifacts_dir: Path) -> List[ContractHand """ pass - def extract_logic_contracts(self) -> List[ContractHandle]: - """Extract logic contracts from build artifacts with config fallback.""" - # Get default artifact directory from manager + def resolve_artifacts_dir(self) -> Path: + """Return the directory this project's build artifacts belong in. + + The directory is not promised to exist: an unbuilt project has artifacts nowhere, + and naming the directory its build *would* write to is what lets the caller say + which build to run. Callers test the path before reading it. + + The default directory name is only a guess. A project can configure its output + elsewhere (Foundry's ``out``, Hardhat's ``paths.artifacts``) and still have a + directory by the default name, for example as the *parent* of the configured one: + ``out = "out/foundry"`` leaves a bare ``out/`` that exists and holds nothing the + extractor can read. So the default only counts while it actually holds artifacts, + and the config decides otherwise. + + The default is asked first for two reasons. It is the cheap question — reading the + config runs ``forge remappings`` for Foundry and loads the project's config through + node for Hardhat and Truffle, none of it memoized. And for a Hardhat project picked + over a Foundry config sitting beside it, the default directory is the evidence that + decided the pick: ``BuildSystemDetector`` ranks the Hardhat side by ``artifacts/`` + without ever reading a Hardhat config. Preferring a configured directory here could + send the extractor somewhere the detector never looked, while the populated directory + that decided the pick goes unread. + + A default directory that exists but holds no artifacts is still the right answer when + the config offers nothing better: it means the build produced nothing, which is what + the caller should report on. With nothing on disk either way there is no artifact to + read, so the answer is the directory this project writes to — the configured one when + the config named one. + """ default_artifact_dir = self.project_root / self.manager.get_default_artifact_dir() + if self.manager.holds_artifacts(default_artifact_dir): + return default_artifact_dir + + configured_dir = self._try_read_artifact_dir_from_config() + if configured_dir is not None and configured_dir.is_dir(): + return configured_dir - artifacts_dir = default_artifact_dir + if default_artifact_dir.is_dir(): + return default_artifact_dir - # Config file fallback: use manager's auto_detect_config() - if not artifacts_dir.exists(): - artifacts_dir = self._try_read_artifact_dir_from_config() - if not artifacts_dir or not artifacts_dir.exists(): - raise Exception( - f"{self.manager.component} artifacts directory '{artifacts_dir}' does not exist. " - f"Please run '{self.manager.get_build_command(profile=self.profile)}' first." - ) + return configured_dir if configured_dir is not None else default_artifact_dir + def extract_logic_contracts(self) -> List[ContractHandle]: + """Extract logic contracts from build artifacts, or name the build that has to run.""" + artifacts_dir = self.resolve_artifacts_dir() if not artifacts_dir.is_dir(): - raise Exception(f"'{artifacts_dir}' exists but is not a directory") + # Say which way the directory is unusable: a path that is there but is a file + # needs a different remedy than one that is absent. + state = "is not a directory" if artifacts_dir.exists() else "does not exist" + raise Exception( + f"{self.manager.component} artifacts directory '{artifacts_dir}' {state}. " + f"Please run '{self.manager.get_build_command(profile=self.profile)}' first." + ) # Delegate to build-system-specific implementation return self.extract_logic_contracts_impl(artifacts_dir) diff --git a/certora_autosetup/parsers/build_system_detector.py b/certora_autosetup/parsers/build_system_detector.py index f3379e37..15fc93f5 100644 --- a/certora_autosetup/parsers/build_system_detector.py +++ b/certora_autosetup/parsers/build_system_detector.py @@ -19,6 +19,7 @@ from certora_autosetup.parsers.foundry import FoundryContractExtractor from certora_autosetup.parsers.hardhat import HardhatContractExtractor from certora_autosetup.parsers.truffle import TruffleContractExtractor +from certora_autosetup.utils.project_dir import foundry_artifact_dir, hardhat_artifact_dir @@ -41,12 +42,12 @@ def detect(project_root: Path) -> BuildSystem: Auto-detect build system from project structure. Detection logic (in order of precedence): - 1. Check for foundry.toml → FOUNDRY - 2. Check for hardhat.config.js or hardhat.config.ts → HARDHAT - 3. Check for truffle-config.js or truffle.js → TRUFFLE - 4. Check for package.json with a hardhat/truffle dependency → HARDHAT / TRUFFLE - 5. Check for artifact directory structures → FOUNDRY, HARDHAT or TRUFFLE - 6. Return UNKNOWN if none found + 1. Check for foundry.toml → FOUNDRY, or hardhat.config.js/.ts → HARDHAT. With both + present the artifacts on disk break the tie (see _pick_foundry_or_hardhat) + 2. Check for truffle-config.js or truffle.js → TRUFFLE + 3. Check for package.json with a hardhat/truffle dependency → HARDHAT / TRUFFLE + 4. Check for artifact directory structures → FOUNDRY, HARDHAT or TRUFFLE + 5. Return UNKNOWN if none found Truffle ranks below the other two throughout: a repo migrating off Truffle keeps its stale truffle-config.js next to the foundry.toml/hardhat.config that now drives it. @@ -71,17 +72,7 @@ def detect(project_root: Path) -> BuildSystem: # Handle case where both are present if foundry_present and hardhat_present: - logger.log( - "Both Foundry and Hardhat detected, defaulting to Foundry", - "WARNING", - "BuildSystemDetector" - ) - logger.log( - "Use --build-system hardhat to override", - "INFO", - "BuildSystemDetector" - ) - return BuildSystem.FOUNDRY + return BuildSystemDetector._pick_foundry_or_hardhat(project_root) if foundry_present: return BuildSystem.FOUNDRY @@ -126,30 +117,23 @@ def detect(project_root: Path) -> BuildSystem: out_dir = project_root / "out" artifacts_dir = project_root / "artifacts" - # Check for Foundry structure - if out_dir.exists() and out_dir.is_dir(): - # Look for typical Foundry structure: out/*.sol/*.json - sol_dirs = [d for d in out_dir.iterdir() if d.is_dir() and d.name.endswith(".sol")] - if sol_dirs: - logger.log( - "Detected Foundry from out/ directory structure", - "INFO", - "BuildSystemDetector" - ) - return BuildSystem.FOUNDRY + # Check for Foundry structure: out/*.sol/*.json + if FoundryManager.holds_artifacts(out_dir): + logger.log( + "Detected Foundry from out/ directory structure", + "INFO", + "BuildSystemDetector" + ) + return BuildSystem.FOUNDRY # Check for Hardhat structure - if artifacts_dir.exists() and artifacts_dir.is_dir(): - # Look for Hardhat-specific structure - contracts_dir = artifacts_dir / "contracts" - build_info_dir = artifacts_dir / "build-info" - if contracts_dir.exists() or build_info_dir.exists(): - logger.log( - "Detected Hardhat from artifacts/ directory structure", - "INFO", - "BuildSystemDetector" - ) - return BuildSystem.HARDHAT + if HardhatManager.holds_artifacts(artifacts_dir): + logger.log( + "Detected Hardhat from artifacts/ directory structure", + "INFO", + "BuildSystemDetector" + ) + return BuildSystem.HARDHAT # Check for Truffle structure: build/contracts/.json truffle_build_dir = project_root / "build" / "contracts" @@ -163,6 +147,59 @@ def detect(project_root: Path) -> BuildSystem: return BuildSystem.UNKNOWN + @staticmethod + def _pick_foundry_or_hardhat(project_root: Path) -> BuildSystem: + """ + Choose between a Foundry and a Hardhat config sitting side by side, on artifacts. + + Both configs in one directory says nothing about which one built the tree: a Hardhat + project may keep a foundry.toml that only builds its forge tests, a Foundry project + may keep a Hardhat config for deployment scripts, and a half-finished migration + leaves both behind. Whichever one's artifact directory holds output is the one that + ran, so that is the one whose artifacts the extractor can read. + + Foundry keeps the tie whenever it has artifacts of its own, and when neither side + has any — an unbuilt tree offers no evidence to rank them by. + + Two limits worth knowing. The Foundry side is read from `[profile.default].out`, so a + tree built under a non-default `FOUNDRY_PROFILE` whose `out` differs reads as having no + artifacts. And a Hardhat project whose foundry.toml governs a forge *test* harness looks + like Foundry as soon as that harness is compiled, since the harness fills `out/` too. + + Args: + project_root: Directory holding both config files + + Returns: + BuildSystem.HARDHAT if only Hardhat's artifacts are on disk, else FOUNDRY + """ + # Import logger here to avoid circular dependency + from certora_autosetup.utils.logger import logger + + foundry_out = foundry_artifact_dir(project_root) + hardhat_out = hardhat_artifact_dir(project_root) + foundry_built = foundry_out is not None and FoundryManager.holds_artifacts(foundry_out) + hardhat_built = hardhat_out is not None and HardhatManager.holds_artifacts(hardhat_out) + + if hardhat_built and not foundry_built: + logger.log( + "Both Foundry and Hardhat detected; only Hardhat has build artifacts, using Hardhat", + "INFO", + "BuildSystemDetector" + ) + return BuildSystem.HARDHAT + + logger.log( + "Both Foundry and Hardhat detected, defaulting to Foundry", + "WARNING", + "BuildSystemDetector" + ) + logger.log( + "Use --build-system hardhat to override", + "INFO", + "BuildSystemDetector" + ) + return BuildSystem.FOUNDRY + @staticmethod def resolve(project_root: Path, requested: Optional[str]) -> BuildSystem: """ @@ -170,7 +207,8 @@ def resolve(project_root: Path, requested: Optional[str]) -> BuildSystem: Centralizes the "explicit override or auto-detect" logic so call sites cannot accidentally drop the user's --build-system choice and fall back to detection - (which warns and defaults to Foundry when both build systems are present). + (which, when both build systems are present, picks the one whose artifacts are on + disk — see `_pick_foundry_or_hardhat`). """ if requested is None or requested == "auto": return BuildSystemDetector.detect(project_root) diff --git a/certora_autosetup/parsers/foundry.py b/certora_autosetup/parsers/foundry.py index bbf78e39..6e22876c 100644 --- a/certora_autosetup/parsers/foundry.py +++ b/certora_autosetup/parsers/foundry.py @@ -39,13 +39,9 @@ def build_source_path_to_contracts_map(self) -> Dict[str, List[tuple]]: Applies the same filters as extract_logic_contracts_impl: non-empty bytecode only, excludes libraries/deps/tests. """ - artifacts_dir = self.project_root / self.manager.get_default_artifact_dir() - if not artifacts_dir.exists(): - fallback = self._try_read_artifact_dir_from_config() - if fallback and fallback.exists(): - artifacts_dir = fallback - else: - return {} + artifacts_dir = self.resolve_artifacts_dir() + if not artifacts_dir.is_dir(): + return {} library_files = find_all_library_files_and_names() library_files = {Path(file).stem: names for file, names in library_files.items()} diff --git a/certora_autosetup/setup/auto_munges.py b/certora_autosetup/setup/auto_munges.py index 5aa9b68a..ce37d882 100644 --- a/certora_autosetup/setup/auto_munges.py +++ b/certora_autosetup/setup/auto_munges.py @@ -10,9 +10,9 @@ import traceback from dataclasses import asdict, dataclass from pathlib import Path -from typing import Any, Callable, Dict, List, Tuple +from typing import Callable, Dict, Iterator, List, Tuple -from certora_autosetup.utils.file_utils import stream_ast_files +from certora_autosetup.solidity_ast import AstDump, MemberAccess, SourceAst, iter_nodes_of_type, parse_src from certora_autosetup.utils.scope import Scope CODE_ACCESS_PATCH_FILE = ".certora_internal/code_access_patches.json" @@ -213,6 +213,36 @@ def _load_ast_parent_graph(graph_path: Path) -> Dict[str, Dict[str, Dict[str, st return {} +@dataclass(frozen=True) +class _CodeAccessCandidate: + """The load-bearing fields of a `.code` MemberAccess, whether it came from the + typed tree or from the raw flat-map sweep of nodes the models could not reach.""" + + node_id: str + src: str + expr_src: str + + +def _iter_code_accesses(source: SourceAst) -> Iterator[_CodeAccessCandidate]: + """`.code` MemberAccess candidates of one source, with exact-parity coverage: + typed nodes first, then raw flat-map nodes the typed walk did not reach (nested + under an unknown node type, or the whole source unparsable) — so a solc surprise + cannot hide a .code access. Vyper sources yield nothing.""" + for node in iter_nodes_of_type(source, MemberAccess): + if isinstance(node, MemberAccess): + if node.memberName == 'code': + yield _CodeAccessCandidate( + node_id=str(node.id), src=node.src, expr_src=node.expression.src + ) + elif node.get('memberName') == 'code': + expression = node.get('expression', {}) + yield _CodeAccessCandidate( + node_id=str(node.get('id')), + src=node.get('src', ''), + expr_src=expression.get('src', '') if isinstance(expression, dict) else '', + ) + + def detect_code_accesses(log_func: Callable, ast_path: Path, ast_graph_path: Path, scope: Scope) -> None: """ Detect .code accesses in the AST and create patches to rewrite them as loadCode(pointer) calls. @@ -237,12 +267,14 @@ def detect_code_accesses(log_func: Callable, ast_path: Path, ast_graph_path: Pat patches = [] # Structure: dict[relative_path: dict[absolute_path: dict[node_id: node_data]]] - for relative_path, path_data in stream_ast_files(ast_path): - # Skip files not in scope - if not scope.is_file_in_scope(Path(relative_path)): - continue - - for absolute_path, nodes in path_data.items(): + # Streamed one compilation unit at a time (the dump can be multi-GB); units + # whose main file is out of scope are skipped before any validation work. + for file_asts in AstDump.stream_units( + ast_path, unit_filter=lambda rel: scope.is_file_in_scope(Path(rel)) + ): + relative_path = file_asts.original_file + + for absolute_path, source in file_asts.sources.items(): # Convert absolute path to relative for scope checking # The scope object works with paths relative to project_root try: @@ -261,93 +293,80 @@ def detect_code_accesses(log_func: Callable, ast_path: Path, ast_graph_path: Pat if not scope.is_file_in_scope(rel_path_for_scope): continue - # Iterate through all nodes (they're already flattened) - for _, node in nodes.items(): - if not isinstance(node, dict): + # The node's src offsets refer to THIS source file (not the unit's + # main file), so patches read and target it; the src file_id is not + # needed since the file is already known here. + patch_target = str(rel_path_for_scope) + + # Find MemberAccess nodes with memberName="code" in this source + for candidate in _iter_code_accesses(source): + node_id = candidate.node_id + + # Check if this .code access is used as expression in another node using parent graph + # If the parent graph exists, use it for O(1) lookup + if parent_graph: + parent_map = parent_graph.get(relative_path, {}).get(absolute_path, {}) + parent_id = parent_map.get(node_id) + + if parent_id: + # The raw flat map covers parsed and unparsed sources alike + parent_node = source.raw.get(parent_id, {}) + parent_type = parent_node.get('nodeType') + + # Skip if parent is MemberAccess (like .code.length) or FunctionCall (like x.code()) + if parent_type in ['MemberAccess', 'FunctionCall']: + continue + else: + # Fallback: check manually if graph not available + is_chained = False + for other_node in source.raw.values(): + if not isinstance(other_node, dict): + continue + + # Check if used in MemberAccess (like .code.length) or FunctionCall (like x.code()) + if other_node.get('nodeType') in ['MemberAccess', 'FunctionCall']: + expr = other_node.get('expression', {}) + if str(expr.get('id')) == node_id: + is_chained = True + break + + if is_chained: + continue # Skip this .code access, it's part of a chain or function call + + # Extract source location: "offset:length:file_id" (byte offsets) + if not candidate.src or not candidate.expr_src: + continue + + try: + offset, length, _ = parse_src(candidate.src) + expr_offset, expr_length, _ = parse_src(candidate.expr_src) + except ValueError: continue - # Check if this is a MemberAccess node with memberName="code" - if node.get('nodeType') == 'MemberAccess' and node.get('memberName') == 'code': - node_id = str(node.get('id')) - - # Check if this .code access is used as expression in another node using parent graph - # If the parent graph exists, use it for O(1) lookup - if parent_graph: - parent_map = parent_graph.get(relative_path, {}).get(absolute_path, {}) - parent_id = parent_map.get(node_id) - - if parent_id: - parent_node = nodes.get(parent_id, {}) - parent_type = parent_node.get('nodeType') - - # Skip if parent is MemberAccess (like .code.length) or FunctionCall (like x.code()) - if parent_type in ['MemberAccess', 'FunctionCall']: - continue - else: - # Fallback: check manually if graph not available - is_chained = False - for _, other_node in nodes.items(): - if not isinstance(other_node, dict): - continue - - # Check if used in MemberAccess (like .code.length) or FunctionCall (like x.code()) - if other_node.get('nodeType') in ['MemberAccess', 'FunctionCall']: - expr = other_node.get('expression', {}) - if str(expr.get('id')) == node_id: - is_chained = True - break - - if is_chained: - continue # Skip this .code access, it's part of a chain or function call - - # Extract source location: "offset:length:file_id" - src = node.get('src', '') - if not src: - continue - - parts = src.split(':') - if len(parts) != 3: - continue - - offset = int(parts[0]) - length = int(parts[1]) - # file_id = int(parts[2]) # Not needed since we already know the file from absolute_path - - # Get the expression being accessed (e.g., "pointer" from "pointer.code") - expression = node.get('expression', {}) - expr_src = expression.get('src', '') - if not expr_src: - continue - - expr_parts = expr_src.split(':') - if len(expr_parts) != 3: - continue - - expr_offset = int(expr_parts[0]) - expr_length = int(expr_parts[1]) - - # Read the original expression from the source file - try: - with open(relative_path, 'r') as src_file: - src_content = src_file.read() - expr_text = src_content[expr_offset:expr_offset + expr_length] - original_text = src_content[offset:offset + length] - - # Create replacement: "certora_loadCode(expression)" - replacement = f"certora_loadCode({expr_text})" - - patches.append( - CodeAccessPatch( - file=relative_path, - offset=offset, - length=length, - original=original_text, - replacement=replacement, - ) - ) - - except Exception as e: - log_func(f"Warning: Failed to read source for patch at {relative_path}: {e}", "WARNING") + # Read the original expression from the source file + try: + with open(patch_target, 'r') as src_file: + src_content = src_file.read() + expr_text = src_content[expr_offset:expr_offset + expr_length] + original_text = src_content[offset:offset + length] + + # Create replacement: "certora_loadCode(expression)" + replacement = f"certora_loadCode({expr_text})" + + patch = CodeAccessPatch( + file=patch_target, + offset=offset, + length=length, + original=original_text, + replacement=replacement, + ) + # The same source can appear under several compilation + # units; apply must see each patch once + if patch not in patches: + patches.append(patch) + + except Exception as e: + log_func(f"Warning: Failed to read source for patch at {patch_target}: {e}", "WARNING") if not patches: log_func("✓ No .code accesses found") diff --git a/certora_autosetup/setup/setup_prover.py b/certora_autosetup/setup/setup_prover.py index 257929a5..7f037461 100644 --- a/certora_autosetup/setup/setup_prover.py +++ b/certora_autosetup/setup/setup_prover.py @@ -17,7 +17,7 @@ import traceback from dataclasses import dataclass from pathlib import Path -from typing import TYPE_CHECKING, Any, Dict, List, Optional, Set, Tuple +from typing import TYPE_CHECKING, Any, Dict, Iterable, Iterator, List, Optional, Set, Tuple if TYPE_CHECKING: from certora_autosetup.setup.setup_summaries import SummarySetup @@ -26,14 +26,26 @@ from certora_autosetup.parsers.build_system_detector import BuildSystem, BuildSystemDetector from certora_autosetup.parsers.foundry import FoundryContractExtractor from certora_autosetup.utils.contract_utils import parse_contract_files +from certora_autosetup.utils.import_diagnostics import ( + UnresolvedImport, + describe_unresolved_imports, +) from certora_autosetup.setup.auto_munges import detect_and_apply_code_access_patches from certora_autosetup.setup.signature_manager import SignatureManager from certora_autosetup.setup.signature_types import ContractInfo from certora_autosetup.setup.solidity_utils import extract_definitions_from_solidity +from certora_autosetup.solidity_ast import ( + AstDump, + ContractDefinition, + FileAsts, + build_parent_graph_json, + iter_nodes_of_type, + stream_raw_units, +) from packaging.version import Version from certora_autosetup.utils.config_manager import convert_solc_version_to_certora_format from certora_autosetup.cache.cache_fs import cache_path, get_fs -from certora_autosetup.utils.file_utils import atomic_write_json_fsspec, stream_ast_files +from certora_autosetup.utils.file_utils import atomic_write_json_fsspec from certora_autosetup.utils.llm_util import ledger_component from certora_autosetup.utils.constants import ( DEFAULT_SOLC_VERSION, @@ -53,6 +65,54 @@ from certora_autosetup.utils.solc_version_resolver import VIA_IR_MIN_VERSION from certora_autosetup.utils.types import ContractHandle, ContractKind, TypeParseMode, parse_type_descriptor +@dataclass(frozen=True) +class _ContractDeclView: + """Uniform view of a ContractDefinition for declaration/inheritance scans, whether + it came from the typed AST or from the raw fallback of an unparsable source.""" + + source_path: str + node_id: Optional[int] + name: str + abstract: bool + contract_kind: str + linearized_base_ids: List[int] + + +def _iter_contract_declarations(units: Iterable[FileAsts]) -> Iterator[_ContractDeclView]: + """Every contract declaration in a stream of compilation units (typically + ``AstDump.stream_units(...)``, so the multi-GB dump is never fully in memory): + typed where the models parsed, completed by a raw flat-map sweep for anything + the typed walk could not reach (a solc surprise cannot hide contracts from + setup; Vyper sources contribute nothing — no ContractDefinition nodes).""" + for file_asts in units: + for source in file_asts.sources.values(): + yield from _unit_contract_declarations(source) + + +def _unit_contract_declarations(source) -> Iterator[_ContractDeclView]: + for node in iter_nodes_of_type(source, ContractDefinition): + if isinstance(node, ContractDefinition): + yield _ContractDeclView( + source_path=source.source_path, + node_id=node.id, + name=node.name, + abstract=node.abstract, + contract_kind=node.contractKind, + linearized_base_ids=list(node.linearizedBaseContracts), + ) + else: + yield _ContractDeclView( + source_path=source.source_path, + node_id=node.get("id"), + name=node.get("name") or "", + abstract=bool(node.get("abstract", False)), + contract_kind=node.get("contractKind", "contract"), + linearized_base_ids=[ + i for i in node.get("linearizedBaseContracts", []) if isinstance(i, int) + ], + ) + + class CompilationAnalysisError(Exception): """Raised when compilation analysis fails.""" @@ -67,27 +127,6 @@ class SummarySetupError(Exception): -@dataclass -class _ContractDef: - """ContractDefinition fields needed to resolve inheritance/abstract info.""" - - id: Optional[int] - name: Optional[str] - abstract: bool - contract_kind: str - linearized_base_contracts: List[int] - - @classmethod - def from_ast_node(cls, node: Dict[str, Any]) -> "_ContractDef": - return cls( - id=node.get("id"), - name=node.get("name"), - abstract=node.get("abstract", False), - contract_kind=node.get("contractKind", "contract"), - linearized_base_contracts=node.get("linearizedBaseContracts", []), - ) - - class SetupProver: """Class to handle setup operations for Certora Prover.""" @@ -130,6 +169,9 @@ def __init__( # Directory holding the contract's build config; assigned by Autosetup alongside # build_system once detection has run. self.build_config_dir: Path = Path.cwd() + # Classification of the unresolved imports the last compilation run hit, carried over + # from the workaround manager so failure messages can name the class. + self.last_import_diagnostics: List[UnresolvedImport] = [] # Track compilation configuration updates self.compilation_config_updates: Dict[str, Any] = {} @@ -431,11 +473,13 @@ def run_compilation_analysis( import_patcher_applied = False raise CompilationAnalysisError( "Compilation analysis failed even after import patch" + + self._import_diagnostics_suffix() ) else: self.log("Import patch failed", "ERROR") raise CompilationAnalysisError( "Compilation analysis failed and import patch could not be applied" + + self._import_diagnostics_suffix() ) self.log("✓ Compilation analysis completed successfully") @@ -615,11 +659,21 @@ def _run_compilation_with_workarounds( solc_default_version=self.solc_default_version, verbose=self.verbose, build_config_dir=self.build_config_dir, + declared_via_ir=bool(getattr(self.build_system_config, "via_ir", False)), ) - return workaround_manager.run_compilation_with_workarounds( + result = workaround_manager.run_compilation_with_workarounds( cmd, config_file, compilation_config, contracts, updated_config_dict ) + # Keep the manager's classification of any unresolved imports: the terminal error text + # in run_compilation_analysis names the failure class, not just the phase that failed. + self.last_import_diagnostics = workaround_manager.last_import_diagnostics + return result + + def _import_diagnostics_suffix(self) -> str: + """Trailing block naming the unresolved imports behind a compilation failure, or "".""" + described = describe_unresolved_imports(self.last_import_diagnostics) + return f"\n{described}" if described else "" def _byte_offset_to_line(self, file_path: str, source_bytes: dict) -> int: """Convert a byte offset in a source file to a 1-based line number.""" @@ -756,16 +810,10 @@ def _build_declared_contracts_by_file(self) -> Dict[str, Set[str]]: ast_path = self._build_dir / FILE_BUILD_ASTS if self._build_dir else None if not ast_path or not ast_path.exists(): return {} - for _relative_path, abs_path_dict in stream_ast_files(ast_path): - for abs_path, nodes in abs_path_dict.items(): - rel = self.scope.get_relative_path(Path(abs_path)) - for node in nodes.values(): - if ( - node.get("nodeType") == "ContractDefinition" - and node.get("contractKind") != "interface" - and node.get("name") - ): - contracts_by_file.setdefault(rel, set()).add(node["name"]) + for decl in _iter_contract_declarations(AstDump.stream_units(ast_path)): + if decl.contract_kind != "interface" and decl.name: + rel = self.scope.get_relative_path(Path(decl.source_path)) + contracts_by_file.setdefault(rel, set()).add(decl.name) return contracts_by_file def _sole_contract_declared_in(self, original_file: str) -> str: @@ -1175,37 +1223,31 @@ def _extract_inheritance_and_abstract_from_ast(self, ast_file_path: Optional[Pat try: self.log(f"Extracting inheritance info from {ast_file_path}") - # Stream the (multi-GB) .asts.json once, keeping only the fields of each - # ContractDefinition needed below so it is never fully materialized. - contract_defs: List[_ContractDef] = [] - for _file_path, abs_path_dict in stream_ast_files(ast_file_path): - for _abs_path, nodes in abs_path_dict.items(): - for _node_id, node in nodes.items(): - if isinstance(node, dict) and node.get("nodeType") == "ContractDefinition": - contract_defs.append(_ContractDef.from_ast_node(node)) + # Stream the (multi-GB) .asts.json once, keeping only the slim per-contract + # views needed below so the dump is never fully materialized. + declarations = list(_iter_contract_declarations(AstDump.stream_units(ast_file_path))) # Build ID to contract name mapping once - id_to_name = {} - for cd in contract_defs: - if cd.id and cd.name: - id_to_name[cd.id] = cd.name + id_to_name = { + decl.node_id: decl.name for decl in declarations if decl.node_id and decl.name + } # Now process contracts and resolve inheritance using the pre-built mapping - for cd in contract_defs: - contract_name = cd.name - if contract_name: - # Check if abstract or interface - if cd.abstract or cd.contract_kind == "interface": - abstract_contracts.add(contract_name) - self.log(f"Identified {'abstract' if cd.abstract else 'interface'}: {contract_name}", "DEBUG") - - # Get linearized base contracts (includes self + all inherited contracts) - linearized = cd.linearized_base_contracts - if len(linearized) > 1: # More than just self - # Convert IDs to contract names using pre-built mapping - base_contracts = [id_to_name[contract_id] for contract_id in linearized[1:] if contract_id in id_to_name] - if base_contracts: - inheritance_info[contract_name] = base_contracts + for decl in declarations: + if not decl.name: + continue + # Check if abstract or interface + if decl.abstract or decl.contract_kind == "interface": + abstract_contracts.add(decl.name) + self.log(f"Identified {'abstract' if decl.abstract else 'interface'}: {decl.name}", "DEBUG") + + # Get linearized base contracts (includes self + all inherited contracts) + linearized = decl.linearized_base_ids + if len(linearized) > 1: # More than just self + # Convert IDs to contract names using pre-built mapping + base_contracts = [id_to_name[contract_id] for contract_id in linearized[1:] if contract_id in id_to_name] + if base_contracts: + inheritance_info[decl.name] = base_contracts self.log(f"Extracted inheritance for {len(inheritance_info)} contracts", "DEBUG") self.log(f"Found {len(abstract_contracts)} abstract/interface contracts to skip", "INFO") @@ -1332,7 +1374,7 @@ def generate_ast_graph(self, ast_path: Path) -> None: ast_path: Path to the .asts.json file Output: - Writes to .certora_internal/.ast_graph.json with structure: + Writes to .certora_internal/all_ast_parent_graph.json with structure: { "relative_path": { "absolute_path": { @@ -1344,25 +1386,10 @@ def generate_ast_graph(self, ast_path: Path) -> None: self.log("Building AST parent graph...") try: - # Build parent graph: node_id -> parent_node_id - parent_graph = {} - - # Structure: dict[relative_path: dict[absolute_path: dict[node_id: node_data]]] - for relative_path, path_data in stream_ast_files(ast_path): - parent_graph[relative_path] = {} - - for absolute_path, nodes in path_data.items(): - parent_graph[relative_path][absolute_path] = {} - - # For each node, find all child node IDs and map them to this parent - for node_id, node in nodes.items(): - if not isinstance(node, dict): - continue - - # Find all child node IDs referenced in this node - child_ids = self._extract_child_node_ids(node) - for child_id in child_ids: - parent_graph[relative_path][absolute_path][str(child_id)] = str(node_id) + # Build parent graph: node_id -> parent_node_id (byte-compatible legacy + # format, streamed one compilation unit at a time — the dump can be + # multi-GB) + parent_graph = build_parent_graph_json(stream_raw_units(ast_path)) # Write parent graph to JSON graph_path = self.getASTParentGraphPath() @@ -1375,30 +1402,6 @@ def generate_ast_graph(self, ast_path: Path) -> None: self.log(f"Warning: Failed to generate AST parent graph: {e}", "WARNING") self.log(f"Traceback: {traceback.format_exc()}", "WARNING") - def _extract_child_node_ids(self, node: Any) -> List[int]: - """ - Extract all child node IDs from an AST node. - - Args: - node: AST node (dict or other type) - - Returns: - List of child node IDs - """ - child_ids = [] - - if isinstance(node, dict): - for key, value in node.items(): - # Look for 'id' fields in nested structures - if isinstance(value, dict) and 'id' in value: - child_ids.append(value['id']) - elif isinstance(value, list): - for item in value: - if isinstance(item, dict) and 'id' in item: - child_ids.append(item['id']) - - return child_ids - def getASTPath(self) -> Path: return Path(".certora_internal/all_asts.json") diff --git a/certora_autosetup/solidity_ast/__init__.py b/certora_autosetup/solidity_ast/__init__.py new file mode 100644 index 00000000..8c870d12 --- /dev/null +++ b/certora_autosetup/solidity_ast/__init__.py @@ -0,0 +1,173 @@ +"""Typed pydantic models of the Solidity compact AST as dumped by +``certoraRun --dump_asts`` (see ``base.py`` for the modeling conventions and +``loader.py`` for the dump structure and degradation policy). + +Typical use:: + + from certora_autosetup.solidity_ast import AstDump, ContractDefinition, find_all + + dump = AstDump.load(".certora_internal/all_asts.json") + for _, _, source_unit in dump.iter_parsed_roots(): + for contract in find_all(source_unit, ContractDefinition): + ... +""" + +__all__ = [ + # base + "AstNode", "SolcNode", "YulNode", "UnknownNode", "SrcLocation", "parse_src", + "TypeDescriptions", "Visibility", "StateMutability", "Mutability", "StorageLocation", + # loader + "AstDump", "FileAsts", "SourceAst", "iter_nodes_of_type", "stream_raw_units", + # traversal + "iter_children", "walk", "find_all", "build_node_index", "build_parent_map", + "build_parent_graph_json", + # unions + "unions", "MODEL_BY_SCHEMA_DEF", "Node", "Expression", "Statement", "TypeName", + "SourceUnitNode", "ContractBodyNode", + # types + "ArrayTypeName", "ElementaryTypeName", "FunctionTypeName", "IdentifierPath", + "Mapping", "UserDefinedTypeName", + # expressions + "Assignment", "BinaryOperation", "Conditional", "ElementaryTypeNameExpression", + "FunctionCall", "FunctionCallOptions", "Identifier", "IndexAccess", "IndexRangeAccess", + "Literal", "MemberAccess", "NewExpression", "TupleExpression", "UnaryOperation", + # statements + "Block", "Break", "Continue", "DoWhileStatement", "EmitStatement", "ExpressionStatement", + "ForStatement", "IfStatement", "InlineAssembly", "PlaceholderStatement", "Return", + "RevertStatement", "TryCatchClause", "TryStatement", "UncheckedBlock", + "VariableDeclarationStatement", "WhileStatement", + # declarations + "ContractDefinition", "EnumDefinition", "EnumValue", "ErrorDefinition", "EventDefinition", + "FunctionDefinition", "ImportDirective", "InheritanceSpecifier", "ModifierDefinition", + "ModifierInvocation", "OverrideSpecifier", "ParameterList", "PragmaDirective", "SourceUnit", + "StorageLayoutSpecifier", "StructDefinition", "StructuredDocumentation", + "UserDefinedValueTypeDefinition", "UsingForDirective", "VariableDeclaration", + # yul + "YulAssignment", "YulBlock", "YulBreak", "YulCase", "YulContinue", "YulExpression", + "YulExpressionStatement", "YulForLoop", "YulFunctionCall", "YulFunctionDefinition", + "YulIdentifier", "YulIf", "YulLeave", "YulLiteral", "YulLiteralHexValue", + "YulLiteralValue", "YulStatement", "YulSwitch", "YulTypedName", "YulVariableDeclaration", +] + +# Importing .unions resolves all cross-module forward references and rebuilds the +# models; loader imports it, and the explicit re-import keeps the ordering obvious. +from . import unions as unions +from . import yul as yul +from .base import ( + AstNode, + Mutability, + SolcNode, + SrcLocation, + StateMutability, + StorageLocation, + TypeDescriptions, + UnknownNode, + Visibility, + YulNode, + parse_src, +) +from .declarations import ( + ContractDefinition, + EnumDefinition, + EnumValue, + ErrorDefinition, + EventDefinition, + FunctionDefinition, + ImportDirective, + InheritanceSpecifier, + ModifierDefinition, + ModifierInvocation, + OverrideSpecifier, + ParameterList, + PragmaDirective, + SourceUnit, + StorageLayoutSpecifier, + StructDefinition, + StructuredDocumentation, + UserDefinedValueTypeDefinition, + UsingForDirective, + VariableDeclaration, +) +from .expressions import ( + Assignment, + BinaryOperation, + Conditional, + ElementaryTypeNameExpression, + FunctionCall, + FunctionCallOptions, + Identifier, + IndexAccess, + IndexRangeAccess, + Literal, + MemberAccess, + NewExpression, + TupleExpression, + UnaryOperation, +) +from .loader import AstDump, FileAsts, SourceAst, iter_nodes_of_type, stream_raw_units +from .statements import ( + Block, + Break, + Continue, + DoWhileStatement, + EmitStatement, + ExpressionStatement, + ForStatement, + IfStatement, + InlineAssembly, + PlaceholderStatement, + Return, + RevertStatement, + TryCatchClause, + TryStatement, + UncheckedBlock, + VariableDeclarationStatement, + WhileStatement, +) +from .traversal import ( + build_node_index, + build_parent_graph_json, + build_parent_map, + find_all, + iter_children, + walk, +) +from .types import ( + ArrayTypeName, + ElementaryTypeName, + FunctionTypeName, + IdentifierPath, + Mapping, + UserDefinedTypeName, +) +from .unions import ( + MODEL_BY_SCHEMA_DEF, + ContractBodyNode, + Expression, + Node, + SourceUnitNode, + Statement, + TypeName, +) +from .yul import ( + YulAssignment, + YulBlock, + YulBreak, + YulCase, + YulContinue, + YulExpression, + YulExpressionStatement, + YulForLoop, + YulFunctionCall, + YulFunctionDefinition, + YulIdentifier, + YulIf, + YulLeave, + YulLiteral, + YulLiteralHexValue, + YulLiteralValue, + YulStatement, + YulSwitch, + YulTypedName, + YulVariableDeclaration, +) diff --git a/certora_autosetup/solidity_ast/__main__.py b/certora_autosetup/solidity_ast/__main__.py new file mode 100644 index 00000000..ce23e590 --- /dev/null +++ b/certora_autosetup/solidity_ast/__main__.py @@ -0,0 +1,134 @@ +"""Summarize a ``.asts.json`` dump through the typed models. + +Usage:: + + python -m certora_autosetup.solidity_ast [--json] [--solc-version V] + +Per source file: parse status, node counts, unknown node types, unmodeled fields, +and round-trip fidelity (does the typed tree re-serialize to the exact source +JSON?) — a quick way to validate the models against a real project's dump. The +dump is streamed one compilation unit at a time, so multi-GB dumps stay cheap. +``--json`` emits one machine-readable summary object instead of text. +""" + +import argparse +import json +import sys +from collections import Counter +from typing import Any + +from .base import UnknownNode +from .declarations import ContractDefinition +from .diagnostics import roundtrip_diffs +from .loader import AstDump, FileAsts, SourceAst +from .traversal import find_all, walk + + +def _source_report(source_ast: SourceAst) -> dict[str, Any]: + if source_ast.root is None: + return { + "status": source_ast.raw_kind, + "error": source_ast.parse_error, + "raw_nodes": len(source_ast.raw), + } + nodes = list(walk(source_ast.root)) + unknown = Counter(n.nodeType for n in nodes if isinstance(n, UnknownNode)) + extras = Counter( + f"{type(n).__name__}.{key}" for n in nodes for key in (n.model_extra or {}) + ) + diffs = roundtrip_diffs(source_ast) + return { + "status": "ok", + "nodes": len(nodes), + "indexed": len(source_ast.nodes), + "unknown_node_types": dict(unknown), + "unmodeled_fields": dict(extras), + "roundtrip_diffs": diffs, + } + + +def _print_unit_text(file_asts: FileAsts, per_file: dict[str, dict[str, Any]]) -> None: + print(file_asts.original_file) + id_to_name = { + c.id: c.name + for source in file_asts.sources.values() + if source.root is not None + for c in find_all(source.root, ContractDefinition) + } + for source in file_asts.sources.values(): + r = per_file[source.source_path] + if r["status"] != "ok": + detail = f": {r['error']}" if r.get("error") else "" + print(f" {source.source_path} [{r['status']}]{detail}") + continue + print(f" {source.source_path} [ok] {r['nodes']} nodes, {r['indexed']} with ids") + if r["unknown_node_types"]: + print(f" unknown node types: {r['unknown_node_types']}") + if r["unmodeled_fields"]: + print(f" unmodeled fields: {r['unmodeled_fields']}") + if r["roundtrip_diffs"]: + print(f" roundtrip diffs ({len(r['roundtrip_diffs'])}):") + for d in r["roundtrip_diffs"][:10]: + print(f" {d}") + assert source.root is not None + for contract in find_all(source.root, ContractDefinition): + bases = [ + id_to_name.get(i, f"#{i}") for i in contract.linearizedBaseContracts[1:] + ] + abstract = "abstract " if contract.abstract else "" + inherits = f" is {', '.join(bases)}" if bases else "" + print(f" {abstract}{contract.contractKind} {contract.name}{inherits}") + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=(__doc__ or "").partition("\n")[0]) + parser.add_argument("dump", help="path to a .asts.json / all_asts.json file") + parser.add_argument("--json", action="store_true", help="machine-readable output") + parser.add_argument( + "--solc-version", + default=None, + help="compiler version that produced the dump; enables the VERSION_GATES " + "check (absent gated fields fail the source instead of reading as None)", + ) + args = parser.parse_args(argv) + + report: dict[str, dict[str, Any]] = {} + for file_asts in AstDump.stream_units(args.dump, solc_version=args.solc_version): + per_file = { + source.source_path: _source_report(source) + for source in file_asts.sources.values() + } + report[file_asts.original_file] = per_file + if not args.json: + _print_unit_text(file_asts, per_file) + + flat = [r for per_file in report.values() for r in per_file.values()] + ok = [r for r in flat if r["status"] == "ok"] + clean = [ + r + for r in ok + if not r["unknown_node_types"] and not r["unmodeled_fields"] and not r["roundtrip_diffs"] + ] + summary = { + "sources": len(flat), + "parsed": len(ok), + "vyper": sum(r["status"] == "vyper" for r in flat), + "parse_failed": sum(r["status"] == "parse_failed" for r in flat), + "fully_clean": len(clean), + } + + if args.json: + json.dump({"summary": summary, "files": report}, sys.stdout, indent=1) + print() + else: + print( + f"\n{summary['parsed']}/{summary['sources']} sources parsed, " + f"{summary['fully_clean']} fully clean (no unknowns, no unmodeled fields, " + f"round-trip exact); vyper: {summary['vyper']}, failed: {summary['parse_failed']}" + ) + + return 0 if summary["parse_failed"] == 0 else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/certora_autosetup/solidity_ast/base.py b/certora_autosetup/solidity_ast/base.py new file mode 100644 index 00000000..b539c6c0 --- /dev/null +++ b/certora_autosetup/solidity_ast/base.py @@ -0,0 +1,134 @@ +"""Base classes and shared value types for the Solidity compact-AST pydantic models. + +The models in this subpackage describe the solc "compact AST" (the ``ast`` entry of +solc standard-json output for each source file), as it appears inside the +``.asts.json`` dump produced by ``certoraRun --dump_asts``. Field sets are +transcribed from the vendored OpenZeppelin ``solidity-ast`` JSON Schema +(see ``schema/schema.json`` and ``schema/NOTICE``), which unions solc versions +>= 0.6 into a single schema. ``tests/solidity_ast/test_schema_conformance.py`` +machine-checks every model against that schema. + +Transcription conventions (uniform across all node modules): + +- schema-required property -> plain annotated field, no default +- schema-optional property (absent + from ``required``; version-gated) -> ``T | None = None`` +- required-but-nullable property + (``anyOf [T, null]``) -> ``T | None`` with NO default +- ``nodeType`` -> ``Literal["X"]`` (the union discriminator) +- reference to a schema union helper + (Expression/Statement/TypeName/...) -> string forward ref to the union alias, + resolved by ``unions.model_rebuild`` wiring +- schema enum property -> shared ``Literal`` alias below when it matches a + helper definition, inline ``Literal[...]`` otherwise +- python-keyword property name -> trailing-underscore field with ``alias=`` (the only + known case is ``UsingForDirective.global``) +""" + +from typing import Callable, Literal, NamedTuple + +from pydantic import BaseModel, ConfigDict + +# Shared enum aliases, mirroring the schema's helper definitions of the same names. +Visibility = Literal["external", "public", "internal", "private"] +StateMutability = Literal["payable", "pure", "nonpayable", "view"] +Mutability = Literal["mutable", "immutable", "constant"] +StorageLocation = Literal["calldata", "default", "memory", "storage", "transient"] + + +class SrcLocation(NamedTuple): + """Decoded solc source location ("offset:length:fileIndex", byte-based).""" + + offset: int + length: int + file_index: int + + +def parse_src(src: str) -> SrcLocation: + """Parse a solc ``src`` string ("offset:length:fileIndex") into byte offsets. + + Raises ValueError on malformed input. ``fileIndex`` may be -1 for nodes solc + synthesizes without a source file. + """ + offset, length, file_index = src.split(":") + return SrcLocation(int(offset), int(length), int(file_index)) + + +class AstNode(BaseModel): + """Common base of every Solidity and Yul compact-AST node. + + ``extra="allow"`` keeps fields from newer solc releases (not yet in the vendored + schema) available via ``model_extra`` instead of failing validation. + """ + + model_config = ConfigDict( + extra="allow", validate_by_name=True, serialize_by_alias=True + ) + + src: str + # Injected by certoraRun into the dump on nodes enclosed in a ContractDefinition; + # never present in raw solc output. + certora_contract_name: str | None = None + + @property + def src_location(self) -> SrcLocation: + return parse_src(self.src) + + +class SolcNode(AstNode): + """A Solidity-language node: always carries a numeric ``id``.""" + + id: int + + +class YulNode(AstNode): + """A Yul node (inside ``InlineAssembly.AST``): no ``id``; ``src`` points into the + original Solidity source and ``nativeSrc`` (solc >= 0.8.21) into the generated Yul. + """ + + nativeSrc: str | None = None + + +class UnknownNode(AstNode): + """Fallback member of every node union: a node whose ``nodeType`` this model set + does not know (newer solc than the vendored schema, or an exotic construct). + + All its fields land in ``model_extra`` and its children stay raw dicts; typed + queries skip it, so an unknown node degrades gracefully instead of failing the + whole source file. + """ + + nodeType: str + id: int | None = None + # Lenient: synthesized nodes may lack src; "" fails any src-offset use downstream + # the same way a missing node would. + src: str = "" + + +UNKNOWN_TAG = "__unknown__" + + +def tag_by_node_type(known: frozenset[str]) -> Callable[[object], str]: + """Discriminator function for a node union: returns the ``nodeType`` tag when it is + one of ``known``, else ``UNKNOWN_TAG`` (routing to the union's UnknownNode member). + Handles both dicts (validation) and model instances (serialization). + """ + + def tag(value: object) -> str: + node_type = ( + value.get("nodeType") + if isinstance(value, dict) + else getattr(value, "nodeType", None) + ) + return node_type if isinstance(node_type, str) and node_type in known else UNKNOWN_TAG + + return tag + + +class TypeDescriptions(BaseModel): + """The ``typeDescriptions`` object attached to expressions and type names.""" + + model_config = ConfigDict(extra="allow") + + typeIdentifier: str | None = None + typeString: str | None = None diff --git a/certora_autosetup/solidity_ast/declarations.py b/certora_autosetup/solidity_ast/declarations.py new file mode 100644 index 00000000..4a649c83 --- /dev/null +++ b/certora_autosetup/solidity_ast/declarations.py @@ -0,0 +1,332 @@ +"""Declaration nodes of the Solidity compact AST (source-unit and contract-body level). + +Also defines ``SourceUnit`` itself, transcribed from the schema root (it is the +schema's top-level object, not a member of ``definitions``). +""" + +from typing import TYPE_CHECKING, Literal + +from pydantic import BaseModel, ConfigDict, Field, field_validator + +from .base import ( + Mutability, + SolcNode, + StateMutability, + StorageLocation, + TypeDescriptions, + Visibility, +) + +if TYPE_CHECKING: + from .expressions import Identifier + from .statements import Block + from .types import IdentifierPath, UserDefinedTypeName + from .unions import ContractBodyNode, Expression, SourceUnitNode, TypeName + + +class StructuredDocumentation(SolcNode): + """A NatSpec documentation node.""" + + text: str + nodeType: Literal["StructuredDocumentation"] + + +class OverrideSpecifier(SolcNode): + """An ``override(...)`` specifier on a function, modifier, or state variable.""" + + overrides: "list[UserDefinedTypeName] | list[IdentifierPath]" + nodeType: Literal["OverrideSpecifier"] + + +class VariableDeclaration(SolcNode): + """A variable declaration: state variable, parameter, struct member, or local.""" + + name: str + nameLocation: str | None = None + baseFunctions: list[int] | None = None + constant: bool + documentation: StructuredDocumentation | None = None + functionSelector: str | None = None + indexed: bool | None = None + # Absent pre-0.6.6 (only `constant` existed); LENIENT_REQUIRED. + mutability: Mutability | None = None + overrides: OverrideSpecifier | None = None + scope: int + stateVariable: bool + storageLocation: StorageLocation + typeDescriptions: TypeDescriptions + typeName: "TypeName | None" = None + value: "Expression | None" = None + visibility: Visibility + nodeType: Literal["VariableDeclaration"] + + @property + def effective_mutability(self) -> Mutability: + """``mutability`` with the pre-0.6.6 case derived from ``constant`` (the only + mutability that existed then besides mutable) — never None.""" + if self.mutability is not None: + return self.mutability + return "constant" if self.constant else "mutable" + + +class ParameterList(SolcNode): + """The parenthesized list of parameters or return values.""" + + parameters: list[VariableDeclaration] + nodeType: Literal["ParameterList"] + + +class EnumValue(SolcNode): + """A single member of an enum definition.""" + + name: str + nameLocation: str | None = None + documentation: StructuredDocumentation | None = None + nodeType: Literal["EnumValue"] + + +class EnumDefinition(SolcNode): + """An ``enum`` definition.""" + + name: str + nameLocation: str | None = None + canonicalName: str + members: list[EnumValue] + documentation: StructuredDocumentation | None = None + nodeType: Literal["EnumDefinition"] + + +class ErrorDefinition(SolcNode): + """A custom ``error`` definition (solc >= 0.8.4).""" + + name: str + nameLocation: str + documentation: StructuredDocumentation | None = None + errorSelector: str | None = None + parameters: ParameterList + nodeType: Literal["ErrorDefinition"] + + +class EventDefinition(SolcNode): + """An ``event`` definition.""" + + name: str + nameLocation: str | None = None + anonymous: bool + eventSelector: str | None = None + documentation: "StructuredDocumentation | str | None" = None + parameters: ParameterList + nodeType: Literal["EventDefinition"] + + +class ModifierInvocation(SolcNode): + """A modifier (or base-constructor) invocation on a function definition.""" + + arguments: "list[Expression] | None" = None + kind: Literal["modifierInvocation", "baseConstructorSpecifier"] | None = None + modifierName: "Identifier | IdentifierPath" + nodeType: Literal["ModifierInvocation"] + + +class ModifierDefinition(SolcNode): + """A ``modifier`` definition.""" + + name: str + nameLocation: str | None = None + baseModifiers: list[int] | None = None + body: "Block | None" = None + documentation: "StructuredDocumentation | str | None" = None + overrides: OverrideSpecifier | None = None + parameters: ParameterList + # `virtual` only exists from solc 0.6, and pre-0.6 everything was implicitly + # overridable — None (unknown), not False; LENIENT_REQUIRED + VERSION_GATES. + virtual: bool | None = None + visibility: Visibility + nodeType: Literal["ModifierDefinition"] + + +class FunctionDefinition(SolcNode): + """A function, constructor, receive/fallback, or free-function definition.""" + + name: str + nameLocation: str | None = None + baseFunctions: list[int] | None = None + body: "Block | None" = None + documentation: "StructuredDocumentation | str | None" = None + functionSelector: str | None = None + implemented: bool + # Absent pre-0.5 (solc 0.4 marks constructors via `isConstructor`); LENIENT_REQUIRED. + kind: Literal["function", "receive", "constructor", "fallback", "freeFunction"] | None = None + modifiers: list[ModifierInvocation] + overrides: OverrideSpecifier | None = None + parameters: ParameterList + returnParameters: ParameterList + scope: int + stateMutability: StateMutability + # `virtual` only exists from solc 0.6, and pre-0.6 everything was implicitly + # overridable — None (unknown), not False; LENIENT_REQUIRED + VERSION_GATES. + virtual: bool | None = None + visibility: Visibility + # Legacy-only fields, not in the schema (FIELD_ALLOWLIST): solc 0.4 flags in + # place of `kind`/`stateMutability`, and the 0.4/0.5 predecessor of + # `baseFunctions`. + isConstructor: bool | None = None + isDeclaredConst: bool | None = None + payable: bool | None = None + superFunction: int | None = None + nodeType: Literal["FunctionDefinition"] + + @property + def effective_kind(self) -> Literal["function", "receive", "constructor", "fallback", "freeFunction"]: + """``kind`` with the solc-0.4 case derived from ``isConstructor``/the empty + name (0.4's unnamed fallback function) — never None.""" + if self.kind is not None: + return self.kind + if self.isConstructor: + return "constructor" + return "fallback" if self.name == "" else "function" + + +class SymbolAlias(BaseModel): + """One ``{symbol as local}`` entry of an ImportDirective's symbolAliases.""" + + model_config = ConfigDict(extra="allow") + + foreign: "Identifier" + local: str | None = None + nameLocation: str | None = None + + +class ImportDirective(SolcNode): + """An ``import`` directive.""" + + absolutePath: str + file: str + nameLocation: str | None = None + scope: int + sourceUnit: int + symbolAliases: list[SymbolAlias] + unitAlias: str + nodeType: Literal["ImportDirective"] + + +class InheritanceSpecifier(SolcNode): + """A base contract in a contract's inheritance list.""" + + arguments: "list[Expression] | None" = None + baseName: "UserDefinedTypeName | IdentifierPath" + nodeType: Literal["InheritanceSpecifier"] + + +class PragmaDirective(SolcNode): + """A ``pragma`` directive; ``literals`` holds its tokenized pieces.""" + + literals: list[str] + nodeType: Literal["PragmaDirective"] + + +class StorageLayoutSpecifier(SolcNode): + """A ``layout at `` storage-layout specifier (solc >= 0.8.29).""" + + baseSlotExpression: "Expression" + nodeType: Literal["StorageLayoutSpecifier"] + + +class StructDefinition(SolcNode): + """A ``struct`` definition.""" + + name: str + nameLocation: str | None = None + canonicalName: str + members: list[VariableDeclaration] + scope: int + visibility: Visibility + documentation: StructuredDocumentation | None = None + nodeType: Literal["StructDefinition"] + + +class UserDefinedValueTypeDefinition(SolcNode): + """A ``type X is `` definition (solc >= 0.8.8).""" + + name: str + nameLocation: str | None = None + canonicalName: str | None = None + underlyingType: "TypeName" + nodeType: Literal["UserDefinedValueTypeDefinition"] + + +class UsingForFunction(BaseModel): + """A plain ``{function: }`` entry of a UsingForDirective's functionList.""" + + model_config = ConfigDict(extra="allow") + + function: "IdentifierPath" + + +class UsingForOperator(BaseModel): + """An ``{operator as }`` entry of a UsingForDirective's functionList.""" + + model_config = ConfigDict(extra="allow") + + operator: Literal[ + "&", "|", "^", "~", "+", "-", "*", "/", "%", "==", "!=", "<", "<=", ">", ">=" + ] + definition: "IdentifierPath" + + +class UsingForDirective(SolcNode): + """A ``using ... for ...`` directive.""" + + functionList: list[UsingForFunction | UsingForOperator] | None = None + global_: bool | None = Field(default=None, alias="global") + libraryName: "UserDefinedTypeName | IdentifierPath | None" = None + typeName: "TypeName | None" = None + nodeType: Literal["UsingForDirective"] + + +class ContractDefinition(SolcNode): + """A contract, interface, or library definition.""" + + name: str + nameLocation: str | None = None + # Schema-required, but the concept only exists from solc 0.6 — a default keeps + # below-floor (0.5.x) sources parseable (lenient-older policy; conformance + # deviation LENIENT_REQUIRED). + abstract: bool = False + baseContracts: list[InheritanceSpecifier] + canonicalName: str | None = None + contractDependencies: list[int] + contractKind: Literal["contract", "interface", "library"] + # NatSpec is a plain string in dumps from solc <= 0.5 (node form from 0.6); + # same widening on Function/Modifier/EventDefinition. DELIBERATELY_OPEN. + documentation: "StructuredDocumentation | str | None" = None + fullyImplemented: bool + linearizedBaseContracts: list[int] + nodes: list["ContractBodyNode"] + scope: int + usedErrors: list[int] | None = None + usedEvents: list[int] | None = None + internalFunctionIDs: dict[str, int] | None = None + storageLayout: StorageLayoutSpecifier | None = None + nodeType: Literal["ContractDefinition"] + + @field_validator("internalFunctionIDs", mode="before") + @classmethod + def _drop_injected_contract_name(cls, value: object) -> object: + # certoraRun's certora_contract_name stamping walks every dict under a + # ContractDefinition, including this plain function-id map; drop the injected + # string entry so the values stay int-typed. + if isinstance(value, dict): + return {k: v for k, v in value.items() if k != "certora_contract_name"} + return value + + +class SourceUnit(SolcNode): + """The root node of one source file's AST (the schema's top-level object).""" + + absolutePath: str + exportedSymbols: dict[str, list[int]] + experimentalSolidity: bool | None = None + license: str | None = None + nodes: list["SourceUnitNode"] + nodeType: Literal["SourceUnit"] diff --git a/certora_autosetup/solidity_ast/diagnostics.py b/certora_autosetup/solidity_ast/diagnostics.py new file mode 100644 index 00000000..fc407a00 --- /dev/null +++ b/certora_autosetup/solidity_ast/diagnostics.py @@ -0,0 +1,68 @@ +"""Fidelity diagnostics: does a typed tree serialize back to the exact source JSON? + +Used by the round-trip tests and by ``python -m certora_autosetup.solidity_ast`` to +validate the models against real dumps at scale. +""" + +from typing import Any + +from .loader import SourceAst + + +def roundtrip_diffs(source: SourceAst, limit: int = 50) -> list[str]: + """Structural differences between ``source.root`` re-serialized and the raw + SourceUnit JSON it was parsed from (empty list == byte-loyal modulo key order). + + ``model_dump(exclude_unset=True)`` keeps exactly the fields present in the input + (absent optional fields stay absent, explicit nulls stay null, unknown fields ride + along in ``model_extra``). The one deliberate normalization is reversed before + comparing: the certoraRun contract-name stamp that lands inside the plain + ``internalFunctionIDs`` map is dropped by the model, so it is dropped from the + raw side too. + """ + if source.root is None: + return [] + raw_root = next( + n + for n in source.raw.values() + if isinstance(n, dict) and n.get("nodeType") == "SourceUnit" + ) + dumped = source.root.model_dump(mode="json", by_alias=True, exclude_unset=True) + expected = _drop_internal_function_id_stamp(raw_root) + diffs: list[str] = [] + _diff(dumped, expected, "", diffs, limit) + return diffs + + +def _drop_internal_function_id_stamp(node: Any) -> Any: + if isinstance(node, dict): + out = {} + for key, value in node.items(): + if key == "internalFunctionIDs" and isinstance(value, dict): + value = {k: v for k, v in value.items() if k != "certora_contract_name"} + out[key] = _drop_internal_function_id_stamp(value) + return out + if isinstance(node, list): + return [_drop_internal_function_id_stamp(item) for item in node] + return node + + +def _diff(dumped: Any, original: Any, path: str, out: list[str], limit: int) -> None: + if len(out) >= limit: + return + if isinstance(dumped, dict) and isinstance(original, dict): + for key in sorted(dumped.keys() | original.keys()): + if key not in original: + out.append(f"{path}.{key}: only in re-serialized output") + elif key not in dumped: + out.append(f"{path}.{key}: lost from the original") + else: + _diff(dumped[key], original[key], f"{path}.{key}", out, limit) + elif isinstance(dumped, list) and isinstance(original, list): + if len(dumped) != len(original): + out.append(f"{path}: list length {len(dumped)} != {len(original)}") + else: + for i, (d, o) in enumerate(zip(dumped, original)): + _diff(d, o, f"{path}[{i}]", out, limit) + elif dumped != original: + out.append(f"{path}: {dumped!r} != {original!r}") diff --git a/certora_autosetup/solidity_ast/expressions.py b/certora_autosetup/solidity_ast/expressions.py new file mode 100644 index 00000000..69456abc --- /dev/null +++ b/certora_autosetup/solidity_ast/expressions.py @@ -0,0 +1,222 @@ +"""Expression nodes of the Solidity compact AST (members of the Expression union).""" + +# The AST node class `Literal` below shadows typing.Literal, so this module uses +# `typing.Literal[...]` for all tag/enum annotations instead of importing the name. +import typing + +from .base import SolcNode, TypeDescriptions + +if typing.TYPE_CHECKING: + from .types import ElementaryTypeName + from .unions import Expression, TypeName + + +class Assignment(SolcNode): + argumentTypes: list[TypeDescriptions] | None = None + isConstant: bool + isLValue: bool + isPure: bool + lValueRequested: bool + typeDescriptions: TypeDescriptions + leftHandSide: "Expression" + operator: typing.Literal[ + "=", "+=", "-=", "*=", "/=", "%=", "|=", "&=", "^=", ">>=", "<<=" + ] + rightHandSide: "Expression" + nodeType: typing.Literal["Assignment"] + + +class BinaryOperation(SolcNode): + argumentTypes: list[TypeDescriptions] | None = None + isConstant: bool + isLValue: bool + isPure: bool + lValueRequested: bool + typeDescriptions: TypeDescriptions + commonType: TypeDescriptions + leftExpression: "Expression" + operator: typing.Literal[ + "+", "-", "*", "/", "%", "**", "&&", "||", "!=", "==", + "<", "<=", ">", ">=", "^", "&", "|", "<<", ">>", + ] + rightExpression: "Expression" + function: int | None = None + nodeType: typing.Literal["BinaryOperation"] + + +class Conditional(SolcNode): + """A ternary ``condition ? trueExpression : falseExpression``.""" + + argumentTypes: list[TypeDescriptions] | None = None + isConstant: bool + isLValue: bool + isPure: bool + lValueRequested: bool + typeDescriptions: TypeDescriptions + condition: "Expression" + falseExpression: "Expression" + trueExpression: "Expression" + nodeType: typing.Literal["Conditional"] + + +class ElementaryTypeNameExpression(SolcNode): + """An elementary type used as an expression, e.g. the callee in ``uint256(x)``.""" + + argumentTypes: list[TypeDescriptions] | None = None + isConstant: bool + isLValue: bool + isPure: bool + lValueRequested: bool + typeDescriptions: TypeDescriptions + # A plain string ("uint256") in dumps from solc <= 0.5; DELIBERATELY_OPEN. + typeName: "ElementaryTypeName | str" + nodeType: typing.Literal["ElementaryTypeNameExpression"] + + +class FunctionCall(SolcNode): + """A call, type conversion, or struct constructor call (see ``kind``).""" + + argumentTypes: list[TypeDescriptions] | None = None + isConstant: bool + isLValue: bool + isPure: bool + lValueRequested: bool + typeDescriptions: TypeDescriptions + arguments: list["Expression"] + expression: "Expression" + kind: typing.Literal["functionCall", "typeConversion", "structConstructorCall"] + names: list[str] + nameLocations: list[str] | None = None + # try/catch only exists from solc 0.6; LENIENT_REQUIRED. + tryCall: bool = False + nodeType: typing.Literal["FunctionCall"] + + +class FunctionCallOptions(SolcNode): + """Call options attached to a callee, e.g. ``f{value: 1, gas: 2}``.""" + + argumentTypes: list[TypeDescriptions] | None = None + isConstant: bool + isLValue: bool | None = None + isPure: bool + lValueRequested: bool + typeDescriptions: TypeDescriptions + expression: "Expression" + names: list[str] + options: list["Expression"] + nodeType: typing.Literal["FunctionCallOptions"] + + +class Identifier(SolcNode): + argumentTypes: list[TypeDescriptions] | None = None + name: str + overloadedDeclarations: list[int] + referencedDeclaration: int | None = None + typeDescriptions: TypeDescriptions + nodeType: typing.Literal["Identifier"] + + +class IndexAccess(SolcNode): + argumentTypes: list[TypeDescriptions] | None = None + isConstant: bool + isLValue: bool + isPure: bool + lValueRequested: bool + typeDescriptions: TypeDescriptions + baseExpression: "Expression" + indexExpression: "Expression | None" = None + nodeType: typing.Literal["IndexAccess"] + + +class IndexRangeAccess(SolcNode): + """An array slice, e.g. ``arr[1:3]`` (calldata arrays only).""" + + argumentTypes: list[TypeDescriptions] | None = None + isConstant: bool + isLValue: bool + isPure: bool + lValueRequested: bool + typeDescriptions: TypeDescriptions + baseExpression: "Expression" + endExpression: "Expression | None" = None + startExpression: "Expression | None" = None + nodeType: typing.Literal["IndexRangeAccess"] + + +class Literal(SolcNode): + """A literal value (number, string, bool, ...); shadows ``typing.Literal`` here.""" + + argumentTypes: list[TypeDescriptions] | None = None + isConstant: bool + isLValue: bool + isPure: bool + lValueRequested: bool + typeDescriptions: TypeDescriptions + hexValue: str + kind: typing.Literal["bool", "number", "string", "hexString", "unicodeString"] + subdenomination: ( + typing.Literal[ + "seconds", "minutes", "hours", "days", "weeks", + "wei", "gwei", "ether", "finney", "szabo", + ] + | None + ) = None + value: str | None = None + nodeType: typing.Literal["Literal"] + + +class MemberAccess(SolcNode): + argumentTypes: list[TypeDescriptions] | None = None + isConstant: bool + # solc 0.7.2 (only) omits isLValue on enum-member accesses — a compiler bug + # window, not an introduction gate; LENIENT_REQUIRED, no VERSION_GATES entry. + isLValue: bool | None = None + isPure: bool + lValueRequested: bool + typeDescriptions: TypeDescriptions + expression: "Expression" + memberName: str + memberLocation: str | None = None + referencedDeclaration: int | None = None + nodeType: typing.Literal["MemberAccess"] + + +class NewExpression(SolcNode): + """A ``new T`` expression (contract creation or dynamic-array allocation).""" + + argumentTypes: list[TypeDescriptions] | None = None + isConstant: bool + isLValue: bool | None = None + isPure: bool + lValueRequested: bool + typeDescriptions: TypeDescriptions + typeName: "TypeName" + nodeType: typing.Literal["NewExpression"] + + +class TupleExpression(SolcNode): + """A tuple or inline array; ``components`` has None holes for omitted entries.""" + + argumentTypes: list[TypeDescriptions] | None = None + isConstant: bool + isLValue: bool + isPure: bool + lValueRequested: bool + typeDescriptions: TypeDescriptions + components: list["Expression | None"] + isInlineArray: bool + nodeType: typing.Literal["TupleExpression"] + + +class UnaryOperation(SolcNode): + argumentTypes: list[TypeDescriptions] | None = None + isConstant: bool + isLValue: bool + isPure: bool + lValueRequested: bool + typeDescriptions: TypeDescriptions + operator: typing.Literal["++", "--", "-", "!", "delete", "~"] + prefix: bool + subExpression: "Expression" + function: int | None = None + nodeType: typing.Literal["UnaryOperation"] diff --git a/certora_autosetup/solidity_ast/loader.py b/certora_autosetup/solidity_ast/loader.py new file mode 100644 index 00000000..961200c0 --- /dev/null +++ b/certora_autosetup/solidity_ast/loader.py @@ -0,0 +1,262 @@ +"""Typed loader for the ``.asts.json`` dump written by ``certoraRun --dump_asts``. + +The dump is a three-level dict ``{original_file: {source_file: {node_id_str: node}}}``: +the outer key is the contract file the compiler was invoked on, the middle key is every +source in that compilation unit, and the inner map is a flat id-index whose values are +the same nodes that also appear nested inside their parents. The loader therefore +validates each source's SourceUnit tree exactly once and derives the id-index by +traversal, keeping the raw flat map alongside for fallback and byte-compatible uses. + +Degradation policy (a project that compiles must never fail because of this loader): +an unrecognized ``nodeType`` becomes an ``UnknownNode`` inside an otherwise-typed tree; +a source whose shape the models reject entirely is kept raw as ``parse_failed``; +Vyper sources (``ast_type``/``node_id`` dialect) are kept raw as ``vyper``. +""" + +import json +import logging +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any, Callable, Iterator, Literal, TypeVar, get_args + +import ijson +from packaging.version import Version +from pydantic import ValidationError + +from . import unions as unions # import resolves forward refs and rebuilds the models +from .base import AstNode +from .declarations import SourceUnit +from .traversal import build_node_index, find_all, walk + +# Fields the models keep lenient (see LENIENT_REQUIRED in the conformance test) but +# that MUST be present in dumps from the solc version that introduced them onward. +# When the caller knows the producing version (autosetup always does), absence at or +# above the gate is a hard error — wrong results are worse than a failed parse. +# Gates err late where the exact introduction is unverified (never crash wrongly on +# a version that genuinely lacked the field); tightened as fixture evidence grows. +VERSION_GATES: dict[str, dict[str, Version]] = { + "ContractDefinition": {"abstract": Version("0.6.0")}, + "FunctionDefinition": {"kind": Version("0.5.0"), "virtual": Version("0.6.0")}, + "ModifierDefinition": {"virtual": Version("0.6.0")}, + "FunctionCall": {"tryCall": Version("0.6.0")}, + "VariableDeclaration": {"mutability": Version("0.6.6")}, + "InlineAssembly": {"AST": Version("0.6.0"), "evmVersion": Version("0.6.2")}, +} + + +def _version_gate_violation(root: AstNode, solc_version: Version) -> str | None: + """First gated field that is absent although the producing solc must emit it.""" + for node in walk(root): + gates = VERSION_GATES.get(type(node).__name__) + if not gates: + continue + for field_name, gate in gates.items(): + if solc_version >= gate and field_name not in node.model_fields_set: + return ( + f"{type(node).__name__}.{field_name} absent, but solc " + f"{solc_version} (>= {gate}) always emits it" + ) + return None + +logger = logging.getLogger(__name__) + +RawKind = Literal["solidity", "vyper", "parse_failed"] +OnError = Literal["raw", "raise"] + + +@dataclass +class SourceAst: + """One source file's AST within a compilation unit.""" + + source_path: str + root: SourceUnit | None + nodes: dict[int, AstNode] = field(default_factory=dict) + raw: dict[str, Any] = field(default_factory=dict) + raw_kind: RawKind = "solidity" + parse_error: str | None = None + + @property + def is_parsed(self) -> bool: + return self.root is not None + + +@dataclass +class FileAsts: + """All source ASTs of one compilation unit (one outer key of the dump).""" + + original_file: str + sources: dict[str, SourceAst] + + +@dataclass +class AstDump: + """The full, typed view of a ``.asts.json`` dump.""" + + files: dict[str, FileAsts] + + @classmethod + def load( + cls, + path: Path | str, + *, + on_error: OnError = "raw", + solc_version: str | Version | None = None, + ) -> "AstDump": + """``solc_version``: the compiler that produced the dump, when known — + enables the VERSION_GATES check (a gated field absent at or above its gate + fails the source instead of silently reading as None).""" + with open(path, "r", encoding="utf-8") as f: + return cls.from_dict(json.load(f), on_error=on_error, solc_version=solc_version) + + @classmethod + def stream_units( + cls, + path: Path | str, + *, + on_error: OnError = "raw", + solc_version: str | Version | None = None, + unit_filter: Callable[[str], bool] | None = None, + ) -> Iterator[FileAsts]: + """Stream the dump one compilation unit (outer key) at a time — the dump can + be multi-GB, and whole-file loading OOMs constrained runs; peak memory here + is bounded by the largest single unit. ``unit_filter`` (on the outer key) + skips units before any validation work. Consumers must drop each unit before + taking the next. + """ + version = Version(solc_version) if isinstance(solc_version, str) else solc_version + for original_file, per_source in stream_raw_units(path): + if unit_filter is not None and not unit_filter(original_file): + continue + yield FileAsts( + original_file=original_file, + sources={ + source_path: _load_source(source_path, flat, on_error, version) + for source_path, flat in per_source.items() + }, + ) + + @classmethod + def from_dict( + cls, + data: dict[str, Any], + *, + on_error: OnError = "raw", + solc_version: str | Version | None = None, + ) -> "AstDump": + version = Version(solc_version) if isinstance(solc_version, str) else solc_version + files = { + original_file: FileAsts( + original_file=original_file, + sources={ + source_path: _load_source(source_path, flat, on_error, version) + for source_path, flat in per_source.items() + }, + ) + for original_file, per_source in data.items() + } + return cls(files=files) + + def iter_sources(self) -> Iterator[tuple[str, SourceAst]]: + """(original_file, SourceAst) over every source, including vyper/failed ones.""" + for file_asts in self.files.values(): + for source in file_asts.sources.values(): + yield file_asts.original_file, source + + def iter_parsed_roots(self) -> Iterator[tuple[str, str, SourceUnit]]: + """(original_file, source_path, SourceUnit) over successfully parsed sources.""" + for original_file, source in self.iter_sources(): + if source.root is not None: + yield original_file, source.source_path, source.root + + def find_node(self, source_path: str, node_id: int) -> AstNode | None: + for file_asts in self.files.values(): + source = file_asts.sources.get(source_path) + if source is not None and node_id in source.nodes: + return source.nodes[node_id] + return None + + +N = TypeVar("N", bound=AstNode) + + +def iter_nodes_of_type(source: SourceAst, model: type[N]) -> Iterator[N | dict[str, Any]]: + """All nodes of one concrete model type in a source: typed instances from the + parsed tree first, then the raw flat-map dicts of matching nodeType that the + typed walk did not reach (nested under an UnknownNode, or the whole source + unparsable). Gives exact-parity coverage with a raw flat-map scan while staying + typed wherever the models reached; callers must accept both shapes. + """ + (node_type,) = get_args(model.model_fields["nodeType"].annotation) + seen: set[int] = set() + if source.root is not None: + for node in find_all(source.root, model): + node_id = getattr(node, "id", None) # Yul models carry no id + if isinstance(node_id, int): + seen.add(node_id) + yield node + for raw_node in source.raw.values(): + if ( + isinstance(raw_node, dict) + and raw_node.get("nodeType") == node_type + and raw_node.get("id") not in seen + ): + yield raw_node + + +def stream_raw_units(path: Path | str) -> Iterator[tuple[str, dict[str, Any]]]: + """Stream raw ``(original_file, {source_path: flat_node_map})`` pairs without any + model validation — for raw-only passes like the legacy parent-graph builder.""" + with open(path, "rb") as f: + yield from ijson.kvitems(f, "") + + +def _load_source( + source_path: str, + flat: dict[str, Any], + on_error: OnError, + solc_version: Version | None = None, +) -> SourceAst: + node_dicts = [n for n in flat.values() if isinstance(n, dict)] + + has_solidity = any("nodeType" in n for n in node_dicts) + has_vyper = any("nodeType" not in n and ("ast_type" in n or "node_id" in n) for n in node_dicts) + if has_vyper and not has_solidity: + return SourceAst(source_path=source_path, root=None, raw=flat, raw_kind="vyper") + + roots = [n for n in node_dicts if n.get("nodeType") == "SourceUnit"] + if len(roots) != 1: + return _failed( + source_path, flat, f"expected exactly one SourceUnit node, found {len(roots)}", on_error + ) + + try: + root = SourceUnit.model_validate(roots[0]) + except ValidationError as e: + if on_error == "raise": + raise + return _failed( + source_path, flat, f"{e.error_count()} validation error(s): {e.errors()[0]}", on_error + ) + + if solc_version is not None: + violation = _version_gate_violation(root, solc_version) + if violation: + return _failed(source_path, flat, f"version-gate violation: {violation}", on_error) + + nodes = build_node_index(root) + missing = [i for i in flat if i.isdigit() and int(i) not in nodes] + if missing: + logger.debug( + "%s: %d raw index ids not reached by typed traversal (first: %s)", + source_path, len(missing), missing[0], + ) + return SourceAst(source_path=source_path, root=root, nodes=nodes, raw=flat) + + +def _failed(source_path: str, flat: dict[str, Any], msg: str, on_error: OnError) -> SourceAst: + if on_error == "raise": + raise ValueError(f"failed to parse AST of {source_path}: {msg}") + logger.warning("falling back to raw AST for %s: %s", source_path, msg) + return SourceAst( + source_path=source_path, root=None, raw=flat, raw_kind="parse_failed", parse_error=msg + ) diff --git a/certora_autosetup/solidity_ast/schema/NOTICE b/certora_autosetup/solidity_ast/schema/NOTICE new file mode 100644 index 00000000..b2a431f1 --- /dev/null +++ b/certora_autosetup/solidity_ast/schema/NOTICE @@ -0,0 +1,17 @@ +schema.json is vendored from the OpenZeppelin `solidity-ast` npm package. + + Package: solidity-ast + Version: 0.4.62 + Source: https://unpkg.com/solidity-ast@0.4.62/schema.json + Project: https://github.com/OpenZeppelin/solidity-ast + License: MIT (Copyright (c) 2020 OpenZeppelin) — see the project repository + for the full license text. + +It is a JSON Schema (draft-06) of the Solidity compiler's compact JSON AST, +unioning solc versions >= 0.6: fields added in later solc releases are optional, +fields whose value can be null are anyOf-nullable. + +To refresh: pick an explicit version and run + curl -sL https://unpkg.com/solidity-ast@/schema.json -o schema.json +then update this NOTICE and re-run tests/solidity_ast/ (the conformance test +will point at any model fields that need to follow). diff --git a/certora_autosetup/solidity_ast/schema/schema.json b/certora_autosetup/solidity_ast/schema/schema.json new file mode 100644 index 00000000..ac728446 --- /dev/null +++ b/certora_autosetup/solidity_ast/schema/schema.json @@ -0,0 +1,4015 @@ +{ + "$schema": "http://json-schema.org/draft-06/schema#", + "title": "SourceUnit", + "type": "object", + "additionalProperties": false, + "properties": { + "id": { + "type": "integer" + }, + "src": { + "$ref": "#/definitions/SourceLocation" + }, + "absolutePath": { + "type": "string" + }, + "exportedSymbols": { + "type": "object", + "additionalProperties": { + "type": "array", + "items": { + "type": "integer" + } + } + }, + "experimentalSolidity": { + "type": "boolean" + }, + "license": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "nodes": { + "type": "array", + "items": { + "anyOf": [ + { + "$ref": "#/definitions/ContractDefinition" + }, + { + "$ref": "#/definitions/EnumDefinition" + }, + { + "$ref": "#/definitions/ErrorDefinition" + }, + { + "$ref": "#/definitions/FunctionDefinition" + }, + { + "$ref": "#/definitions/ImportDirective" + }, + { + "$ref": "#/definitions/PragmaDirective" + }, + { + "$ref": "#/definitions/StructDefinition" + }, + { + "$ref": "#/definitions/UserDefinedValueTypeDefinition" + }, + { + "$ref": "#/definitions/UsingForDirective" + }, + { + "$ref": "#/definitions/VariableDeclaration" + } + ] + } + }, + "nodeType": { + "enum": [ + "SourceUnit" + ] + } + }, + "required": [ + "id", + "src", + "absolutePath", + "exportedSymbols", + "nodes", + "nodeType" + ], + "definitions": { + "SourceLocation": { + "type": "string", + "pattern": "^\\d+:\\d+:\\d+$" + }, + "Mutability": { + "enum": [ + "mutable", + "immutable", + "constant" + ] + }, + "StateMutability": { + "enum": [ + "payable", + "pure", + "nonpayable", + "view" + ] + }, + "StorageLocation": { + "enum": [ + "calldata", + "default", + "memory", + "storage", + "transient" + ] + }, + "Visibility": { + "enum": [ + "external", + "public", + "internal", + "private" + ] + }, + "TypeDescriptions": { + "type": "object", + "additionalProperties": false, + "properties": { + "typeIdentifier": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "typeString": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + } + }, + "required": [] + }, + "Expression": { + "anyOf": [ + { + "$ref": "#/definitions/Assignment" + }, + { + "$ref": "#/definitions/BinaryOperation" + }, + { + "$ref": "#/definitions/Conditional" + }, + { + "$ref": "#/definitions/ElementaryTypeNameExpression" + }, + { + "$ref": "#/definitions/FunctionCall" + }, + { + "$ref": "#/definitions/FunctionCallOptions" + }, + { + "$ref": "#/definitions/Identifier" + }, + { + "$ref": "#/definitions/IndexAccess" + }, + { + "$ref": "#/definitions/IndexRangeAccess" + }, + { + "$ref": "#/definitions/Literal" + }, + { + "$ref": "#/definitions/MemberAccess" + }, + { + "$ref": "#/definitions/NewExpression" + }, + { + "$ref": "#/definitions/TupleExpression" + }, + { + "$ref": "#/definitions/UnaryOperation" + } + ] + }, + "Statement": { + "anyOf": [ + { + "$ref": "#/definitions/Block" + }, + { + "$ref": "#/definitions/Break" + }, + { + "$ref": "#/definitions/Continue" + }, + { + "$ref": "#/definitions/DoWhileStatement" + }, + { + "$ref": "#/definitions/EmitStatement" + }, + { + "$ref": "#/definitions/ExpressionStatement" + }, + { + "$ref": "#/definitions/ForStatement" + }, + { + "$ref": "#/definitions/IfStatement" + }, + { + "$ref": "#/definitions/InlineAssembly" + }, + { + "$ref": "#/definitions/PlaceholderStatement" + }, + { + "$ref": "#/definitions/Return" + }, + { + "$ref": "#/definitions/RevertStatement" + }, + { + "$ref": "#/definitions/TryStatement" + }, + { + "$ref": "#/definitions/UncheckedBlock" + }, + { + "$ref": "#/definitions/VariableDeclarationStatement" + }, + { + "$ref": "#/definitions/WhileStatement" + } + ] + }, + "TypeName": { + "anyOf": [ + { + "$ref": "#/definitions/ArrayTypeName" + }, + { + "$ref": "#/definitions/ElementaryTypeName" + }, + { + "$ref": "#/definitions/FunctionTypeName" + }, + { + "$ref": "#/definitions/Mapping" + }, + { + "$ref": "#/definitions/UserDefinedTypeName" + } + ] + }, + "ArrayTypeName": { + "type": "object", + "additionalProperties": false, + "properties": { + "id": { + "type": "integer" + }, + "src": { + "$ref": "#/definitions/SourceLocation" + }, + "typeDescriptions": { + "$ref": "#/definitions/TypeDescriptions" + }, + "baseType": { + "$ref": "#/definitions/TypeName" + }, + "length": { + "anyOf": [ + { + "$ref": "#/definitions/Expression" + }, + { + "type": "null" + } + ] + }, + "nodeType": { + "enum": [ + "ArrayTypeName" + ] + } + }, + "required": [ + "id", + "src", + "typeDescriptions", + "baseType", + "nodeType" + ] + }, + "Assignment": { + "type": "object", + "additionalProperties": false, + "properties": { + "id": { + "type": "integer" + }, + "src": { + "$ref": "#/definitions/SourceLocation" + }, + "argumentTypes": { + "anyOf": [ + { + "type": "array", + "items": { + "$ref": "#/definitions/TypeDescriptions" + } + }, + { + "type": "null" + } + ] + }, + "isConstant": { + "type": "boolean" + }, + "isLValue": { + "type": "boolean" + }, + "isPure": { + "type": "boolean" + }, + "lValueRequested": { + "type": "boolean" + }, + "typeDescriptions": { + "$ref": "#/definitions/TypeDescriptions" + }, + "leftHandSide": { + "$ref": "#/definitions/Expression" + }, + "operator": { + "enum": [ + "=", + "+=", + "-=", + "*=", + "/=", + "%=", + "|=", + "&=", + "^=", + ">>=", + "<<=" + ] + }, + "rightHandSide": { + "$ref": "#/definitions/Expression" + }, + "nodeType": { + "enum": [ + "Assignment" + ] + } + }, + "required": [ + "id", + "src", + "isConstant", + "isLValue", + "isPure", + "lValueRequested", + "typeDescriptions", + "leftHandSide", + "operator", + "rightHandSide", + "nodeType" + ] + }, + "BinaryOperation": { + "type": "object", + "additionalProperties": false, + "properties": { + "id": { + "type": "integer" + }, + "src": { + "$ref": "#/definitions/SourceLocation" + }, + "argumentTypes": { + "anyOf": [ + { + "type": "array", + "items": { + "$ref": "#/definitions/TypeDescriptions" + } + }, + { + "type": "null" + } + ] + }, + "isConstant": { + "type": "boolean" + }, + "isLValue": { + "type": "boolean" + }, + "isPure": { + "type": "boolean" + }, + "lValueRequested": { + "type": "boolean" + }, + "typeDescriptions": { + "$ref": "#/definitions/TypeDescriptions" + }, + "commonType": { + "$ref": "#/definitions/TypeDescriptions" + }, + "leftExpression": { + "$ref": "#/definitions/Expression" + }, + "operator": { + "enum": [ + "+", + "-", + "*", + "/", + "%", + "**", + "&&", + "||", + "!=", + "==", + "<", + "<=", + ">", + ">=", + "^", + "&", + "|", + "<<", + ">>" + ] + }, + "rightExpression": { + "$ref": "#/definitions/Expression" + }, + "function": { + "type": "integer" + }, + "nodeType": { + "enum": [ + "BinaryOperation" + ] + } + }, + "required": [ + "id", + "src", + "isConstant", + "isLValue", + "isPure", + "lValueRequested", + "typeDescriptions", + "commonType", + "leftExpression", + "operator", + "rightExpression", + "nodeType" + ] + }, + "Block": { + "type": "object", + "additionalProperties": false, + "properties": { + "id": { + "type": "integer" + }, + "src": { + "$ref": "#/definitions/SourceLocation" + }, + "documentation": { + "type": "string" + }, + "statements": { + "anyOf": [ + { + "type": "array", + "items": { + "$ref": "#/definitions/Statement" + } + }, + { + "type": "null" + } + ] + }, + "nodeType": { + "enum": [ + "Block" + ] + } + }, + "required": [ + "id", + "src", + "nodeType" + ] + }, + "Break": { + "type": "object", + "additionalProperties": false, + "properties": { + "id": { + "type": "integer" + }, + "src": { + "$ref": "#/definitions/SourceLocation" + }, + "documentation": { + "type": "string" + }, + "nodeType": { + "enum": [ + "Break" + ] + } + }, + "required": [ + "id", + "src", + "nodeType" + ] + }, + "Conditional": { + "type": "object", + "additionalProperties": false, + "properties": { + "id": { + "type": "integer" + }, + "src": { + "$ref": "#/definitions/SourceLocation" + }, + "argumentTypes": { + "anyOf": [ + { + "type": "array", + "items": { + "$ref": "#/definitions/TypeDescriptions" + } + }, + { + "type": "null" + } + ] + }, + "isConstant": { + "type": "boolean" + }, + "isLValue": { + "type": "boolean" + }, + "isPure": { + "type": "boolean" + }, + "lValueRequested": { + "type": "boolean" + }, + "typeDescriptions": { + "$ref": "#/definitions/TypeDescriptions" + }, + "condition": { + "$ref": "#/definitions/Expression" + }, + "falseExpression": { + "$ref": "#/definitions/Expression" + }, + "trueExpression": { + "$ref": "#/definitions/Expression" + }, + "nodeType": { + "enum": [ + "Conditional" + ] + } + }, + "required": [ + "id", + "src", + "isConstant", + "isLValue", + "isPure", + "lValueRequested", + "typeDescriptions", + "condition", + "falseExpression", + "trueExpression", + "nodeType" + ] + }, + "Continue": { + "type": "object", + "additionalProperties": false, + "properties": { + "id": { + "type": "integer" + }, + "src": { + "$ref": "#/definitions/SourceLocation" + }, + "documentation": { + "type": "string" + }, + "nodeType": { + "enum": [ + "Continue" + ] + } + }, + "required": [ + "id", + "src", + "nodeType" + ] + }, + "ContractDefinition": { + "type": "object", + "additionalProperties": false, + "properties": { + "id": { + "type": "integer" + }, + "src": { + "$ref": "#/definitions/SourceLocation" + }, + "name": { + "type": "string" + }, + "nameLocation": { + "type": "string" + }, + "abstract": { + "type": "boolean" + }, + "baseContracts": { + "type": "array", + "items": { + "$ref": "#/definitions/InheritanceSpecifier" + } + }, + "canonicalName": { + "type": "string" + }, + "contractDependencies": { + "type": "array", + "items": { + "type": "integer" + } + }, + "contractKind": { + "enum": [ + "contract", + "interface", + "library" + ] + }, + "documentation": { + "anyOf": [ + { + "$ref": "#/definitions/StructuredDocumentation" + }, + { + "type": "null" + } + ] + }, + "fullyImplemented": { + "type": "boolean" + }, + "linearizedBaseContracts": { + "type": "array", + "items": { + "type": "integer" + } + }, + "nodes": { + "type": "array", + "items": { + "anyOf": [ + { + "$ref": "#/definitions/EnumDefinition" + }, + { + "$ref": "#/definitions/ErrorDefinition" + }, + { + "$ref": "#/definitions/EventDefinition" + }, + { + "$ref": "#/definitions/FunctionDefinition" + }, + { + "$ref": "#/definitions/ModifierDefinition" + }, + { + "$ref": "#/definitions/StructDefinition" + }, + { + "$ref": "#/definitions/UserDefinedValueTypeDefinition" + }, + { + "$ref": "#/definitions/UsingForDirective" + }, + { + "$ref": "#/definitions/VariableDeclaration" + } + ] + } + }, + "scope": { + "type": "integer" + }, + "usedErrors": { + "type": "array", + "items": { + "type": "integer" + } + }, + "usedEvents": { + "type": "array", + "items": { + "type": "integer" + } + }, + "internalFunctionIDs": { + "type": "object", + "additionalProperties": { + "type": "integer" + } + }, + "storageLayout": { + "$ref": "#/definitions/StorageLayoutSpecifier" + }, + "nodeType": { + "enum": [ + "ContractDefinition" + ] + } + }, + "required": [ + "id", + "src", + "name", + "abstract", + "baseContracts", + "contractDependencies", + "contractKind", + "fullyImplemented", + "linearizedBaseContracts", + "nodes", + "scope", + "nodeType" + ] + }, + "StorageLayoutSpecifier": { + "type": "object", + "additionalProperties": false, + "properties": { + "id": { + "type": "integer" + }, + "src": { + "$ref": "#/definitions/SourceLocation" + }, + "baseSlotExpression": { + "$ref": "#/definitions/Expression" + }, + "nodeType": { + "enum": [ + "StorageLayoutSpecifier" + ] + } + }, + "required": [ + "id", + "src", + "baseSlotExpression", + "nodeType" + ] + }, + "DoWhileStatement": { + "type": "object", + "additionalProperties": false, + "properties": { + "id": { + "type": "integer" + }, + "src": { + "$ref": "#/definitions/SourceLocation" + }, + "documentation": { + "type": "string" + }, + "body": { + "anyOf": [ + { + "$ref": "#/definitions/Block" + }, + { + "$ref": "#/definitions/Statement" + } + ] + }, + "condition": { + "$ref": "#/definitions/Expression" + }, + "nodeType": { + "enum": [ + "DoWhileStatement" + ] + } + }, + "required": [ + "id", + "src", + "body", + "condition", + "nodeType" + ] + }, + "ElementaryTypeName": { + "type": "object", + "additionalProperties": false, + "properties": { + "id": { + "type": "integer" + }, + "src": { + "$ref": "#/definitions/SourceLocation" + }, + "typeDescriptions": { + "$ref": "#/definitions/TypeDescriptions" + }, + "name": { + "type": "string" + }, + "stateMutability": { + "$ref": "#/definitions/StateMutability" + }, + "nodeType": { + "enum": [ + "ElementaryTypeName" + ] + } + }, + "required": [ + "id", + "src", + "typeDescriptions", + "name", + "nodeType" + ] + }, + "ElementaryTypeNameExpression": { + "type": "object", + "additionalProperties": false, + "properties": { + "id": { + "type": "integer" + }, + "src": { + "$ref": "#/definitions/SourceLocation" + }, + "argumentTypes": { + "anyOf": [ + { + "type": "array", + "items": { + "$ref": "#/definitions/TypeDescriptions" + } + }, + { + "type": "null" + } + ] + }, + "isConstant": { + "type": "boolean" + }, + "isLValue": { + "type": "boolean" + }, + "isPure": { + "type": "boolean" + }, + "lValueRequested": { + "type": "boolean" + }, + "typeDescriptions": { + "$ref": "#/definitions/TypeDescriptions" + }, + "typeName": { + "$ref": "#/definitions/ElementaryTypeName" + }, + "nodeType": { + "enum": [ + "ElementaryTypeNameExpression" + ] + } + }, + "required": [ + "id", + "src", + "isConstant", + "isLValue", + "isPure", + "lValueRequested", + "typeDescriptions", + "typeName", + "nodeType" + ] + }, + "EmitStatement": { + "type": "object", + "additionalProperties": false, + "properties": { + "id": { + "type": "integer" + }, + "src": { + "$ref": "#/definitions/SourceLocation" + }, + "documentation": { + "type": "string" + }, + "eventCall": { + "$ref": "#/definitions/FunctionCall" + }, + "nodeType": { + "enum": [ + "EmitStatement" + ] + } + }, + "required": [ + "id", + "src", + "eventCall", + "nodeType" + ] + }, + "EnumDefinition": { + "type": "object", + "additionalProperties": false, + "properties": { + "id": { + "type": "integer" + }, + "src": { + "$ref": "#/definitions/SourceLocation" + }, + "name": { + "type": "string" + }, + "nameLocation": { + "type": "string" + }, + "canonicalName": { + "type": "string" + }, + "members": { + "type": "array", + "items": { + "$ref": "#/definitions/EnumValue" + } + }, + "documentation": { + "anyOf": [ + { + "$ref": "#/definitions/StructuredDocumentation" + }, + { + "type": "null" + } + ] + }, + "nodeType": { + "enum": [ + "EnumDefinition" + ] + } + }, + "required": [ + "id", + "src", + "name", + "canonicalName", + "members", + "nodeType" + ] + }, + "EnumValue": { + "type": "object", + "additionalProperties": false, + "properties": { + "id": { + "type": "integer" + }, + "src": { + "$ref": "#/definitions/SourceLocation" + }, + "name": { + "type": "string" + }, + "nameLocation": { + "type": "string" + }, + "documentation": { + "anyOf": [ + { + "$ref": "#/definitions/StructuredDocumentation" + }, + { + "type": "null" + } + ] + }, + "nodeType": { + "enum": [ + "EnumValue" + ] + } + }, + "required": [ + "id", + "src", + "name", + "nodeType" + ] + }, + "ErrorDefinition": { + "type": "object", + "additionalProperties": false, + "properties": { + "id": { + "type": "integer" + }, + "src": { + "$ref": "#/definitions/SourceLocation" + }, + "name": { + "type": "string" + }, + "nameLocation": { + "type": "string" + }, + "documentation": { + "anyOf": [ + { + "$ref": "#/definitions/StructuredDocumentation" + }, + { + "type": "null" + } + ] + }, + "errorSelector": { + "type": "string" + }, + "parameters": { + "$ref": "#/definitions/ParameterList" + }, + "nodeType": { + "enum": [ + "ErrorDefinition" + ] + } + }, + "required": [ + "id", + "src", + "name", + "nameLocation", + "parameters", + "nodeType" + ] + }, + "EventDefinition": { + "type": "object", + "additionalProperties": false, + "properties": { + "id": { + "type": "integer" + }, + "src": { + "$ref": "#/definitions/SourceLocation" + }, + "name": { + "type": "string" + }, + "nameLocation": { + "type": "string" + }, + "anonymous": { + "type": "boolean" + }, + "eventSelector": { + "type": "string" + }, + "documentation": { + "anyOf": [ + { + "$ref": "#/definitions/StructuredDocumentation" + }, + { + "type": "null" + } + ] + }, + "parameters": { + "$ref": "#/definitions/ParameterList" + }, + "nodeType": { + "enum": [ + "EventDefinition" + ] + } + }, + "required": [ + "id", + "src", + "name", + "anonymous", + "parameters", + "nodeType" + ] + }, + "ExpressionStatement": { + "type": "object", + "additionalProperties": false, + "properties": { + "id": { + "type": "integer" + }, + "src": { + "$ref": "#/definitions/SourceLocation" + }, + "documentation": { + "type": "string" + }, + "expression": { + "$ref": "#/definitions/Expression" + }, + "nodeType": { + "enum": [ + "ExpressionStatement" + ] + } + }, + "required": [ + "id", + "src", + "expression", + "nodeType" + ] + }, + "ForStatement": { + "type": "object", + "additionalProperties": false, + "properties": { + "id": { + "type": "integer" + }, + "src": { + "$ref": "#/definitions/SourceLocation" + }, + "documentation": { + "type": "string" + }, + "body": { + "anyOf": [ + { + "$ref": "#/definitions/Block" + }, + { + "$ref": "#/definitions/Statement" + } + ] + }, + "condition": { + "anyOf": [ + { + "$ref": "#/definitions/Expression" + }, + { + "type": "null" + } + ] + }, + "initializationExpression": { + "anyOf": [ + { + "anyOf": [ + { + "$ref": "#/definitions/ExpressionStatement" + }, + { + "$ref": "#/definitions/VariableDeclarationStatement" + } + ] + }, + { + "type": "null" + } + ] + }, + "loopExpression": { + "anyOf": [ + { + "$ref": "#/definitions/ExpressionStatement" + }, + { + "type": "null" + } + ] + }, + "isSimpleCounterLoop": { + "type": "boolean" + }, + "nodeType": { + "enum": [ + "ForStatement" + ] + } + }, + "required": [ + "id", + "src", + "body", + "nodeType" + ] + }, + "FunctionCall": { + "type": "object", + "additionalProperties": false, + "properties": { + "id": { + "type": "integer" + }, + "src": { + "$ref": "#/definitions/SourceLocation" + }, + "argumentTypes": { + "anyOf": [ + { + "type": "array", + "items": { + "$ref": "#/definitions/TypeDescriptions" + } + }, + { + "type": "null" + } + ] + }, + "isConstant": { + "type": "boolean" + }, + "isLValue": { + "type": "boolean" + }, + "isPure": { + "type": "boolean" + }, + "lValueRequested": { + "type": "boolean" + }, + "typeDescriptions": { + "$ref": "#/definitions/TypeDescriptions" + }, + "arguments": { + "type": "array", + "items": { + "$ref": "#/definitions/Expression" + } + }, + "expression": { + "$ref": "#/definitions/Expression" + }, + "kind": { + "enum": [ + "functionCall", + "typeConversion", + "structConstructorCall" + ] + }, + "names": { + "type": "array", + "items": { + "type": "string" + } + }, + "nameLocations": { + "type": "array", + "items": { + "type": "string" + } + }, + "tryCall": { + "type": "boolean" + }, + "nodeType": { + "enum": [ + "FunctionCall" + ] + } + }, + "required": [ + "id", + "src", + "isConstant", + "isLValue", + "isPure", + "lValueRequested", + "typeDescriptions", + "arguments", + "expression", + "kind", + "names", + "tryCall", + "nodeType" + ] + }, + "FunctionCallOptions": { + "type": "object", + "additionalProperties": false, + "properties": { + "id": { + "type": "integer" + }, + "src": { + "$ref": "#/definitions/SourceLocation" + }, + "argumentTypes": { + "anyOf": [ + { + "type": "array", + "items": { + "$ref": "#/definitions/TypeDescriptions" + } + }, + { + "type": "null" + } + ] + }, + "isConstant": { + "type": "boolean" + }, + "isLValue": { + "type": "boolean" + }, + "isPure": { + "type": "boolean" + }, + "lValueRequested": { + "type": "boolean" + }, + "typeDescriptions": { + "$ref": "#/definitions/TypeDescriptions" + }, + "expression": { + "$ref": "#/definitions/Expression" + }, + "names": { + "type": "array", + "items": { + "type": "string" + } + }, + "options": { + "type": "array", + "items": { + "$ref": "#/definitions/Expression" + } + }, + "nodeType": { + "enum": [ + "FunctionCallOptions" + ] + } + }, + "required": [ + "id", + "src", + "isConstant", + "isPure", + "lValueRequested", + "typeDescriptions", + "expression", + "names", + "options", + "nodeType" + ] + }, + "FunctionDefinition": { + "type": "object", + "additionalProperties": false, + "properties": { + "id": { + "type": "integer" + }, + "src": { + "$ref": "#/definitions/SourceLocation" + }, + "name": { + "type": "string" + }, + "nameLocation": { + "type": "string" + }, + "baseFunctions": { + "type": "array", + "items": { + "type": "integer" + } + }, + "body": { + "anyOf": [ + { + "$ref": "#/definitions/Block" + }, + { + "type": "null" + } + ] + }, + "documentation": { + "anyOf": [ + { + "$ref": "#/definitions/StructuredDocumentation" + }, + { + "type": "null" + } + ] + }, + "functionSelector": { + "type": "string" + }, + "implemented": { + "type": "boolean" + }, + "kind": { + "enum": [ + "function", + "receive", + "constructor", + "fallback", + "freeFunction" + ] + }, + "modifiers": { + "type": "array", + "items": { + "$ref": "#/definitions/ModifierInvocation" + } + }, + "overrides": { + "anyOf": [ + { + "$ref": "#/definitions/OverrideSpecifier" + }, + { + "type": "null" + } + ] + }, + "parameters": { + "$ref": "#/definitions/ParameterList" + }, + "returnParameters": { + "$ref": "#/definitions/ParameterList" + }, + "scope": { + "type": "integer" + }, + "stateMutability": { + "$ref": "#/definitions/StateMutability" + }, + "virtual": { + "type": "boolean" + }, + "visibility": { + "$ref": "#/definitions/Visibility" + }, + "nodeType": { + "enum": [ + "FunctionDefinition" + ] + } + }, + "required": [ + "id", + "src", + "name", + "implemented", + "kind", + "modifiers", + "parameters", + "returnParameters", + "scope", + "stateMutability", + "virtual", + "visibility", + "nodeType" + ] + }, + "FunctionTypeName": { + "type": "object", + "additionalProperties": false, + "properties": { + "id": { + "type": "integer" + }, + "src": { + "$ref": "#/definitions/SourceLocation" + }, + "typeDescriptions": { + "$ref": "#/definitions/TypeDescriptions" + }, + "parameterTypes": { + "$ref": "#/definitions/ParameterList" + }, + "returnParameterTypes": { + "$ref": "#/definitions/ParameterList" + }, + "stateMutability": { + "$ref": "#/definitions/StateMutability" + }, + "visibility": { + "$ref": "#/definitions/Visibility" + }, + "nodeType": { + "enum": [ + "FunctionTypeName" + ] + } + }, + "required": [ + "id", + "src", + "typeDescriptions", + "parameterTypes", + "returnParameterTypes", + "stateMutability", + "visibility", + "nodeType" + ] + }, + "Identifier": { + "type": "object", + "additionalProperties": false, + "properties": { + "id": { + "type": "integer" + }, + "src": { + "$ref": "#/definitions/SourceLocation" + }, + "argumentTypes": { + "anyOf": [ + { + "type": "array", + "items": { + "$ref": "#/definitions/TypeDescriptions" + } + }, + { + "type": "null" + } + ] + }, + "name": { + "type": "string" + }, + "overloadedDeclarations": { + "type": "array", + "items": { + "type": "integer" + } + }, + "referencedDeclaration": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ] + }, + "typeDescriptions": { + "$ref": "#/definitions/TypeDescriptions" + }, + "nodeType": { + "enum": [ + "Identifier" + ] + } + }, + "required": [ + "id", + "src", + "name", + "overloadedDeclarations", + "typeDescriptions", + "nodeType" + ] + }, + "IdentifierPath": { + "type": "object", + "additionalProperties": false, + "properties": { + "id": { + "type": "integer" + }, + "src": { + "$ref": "#/definitions/SourceLocation" + }, + "name": { + "type": "string" + }, + "nameLocations": { + "type": "array", + "items": { + "type": "string" + } + }, + "referencedDeclaration": { + "type": "integer" + }, + "nodeType": { + "enum": [ + "IdentifierPath" + ] + } + }, + "required": [ + "id", + "src", + "name", + "referencedDeclaration", + "nodeType" + ] + }, + "IfStatement": { + "type": "object", + "additionalProperties": false, + "properties": { + "id": { + "type": "integer" + }, + "src": { + "$ref": "#/definitions/SourceLocation" + }, + "documentation": { + "type": "string" + }, + "condition": { + "$ref": "#/definitions/Expression" + }, + "falseBody": { + "anyOf": [ + { + "anyOf": [ + { + "$ref": "#/definitions/Statement" + }, + { + "$ref": "#/definitions/Block" + } + ] + }, + { + "type": "null" + } + ] + }, + "trueBody": { + "anyOf": [ + { + "$ref": "#/definitions/Statement" + }, + { + "$ref": "#/definitions/Block" + } + ] + }, + "nodeType": { + "enum": [ + "IfStatement" + ] + } + }, + "required": [ + "id", + "src", + "condition", + "trueBody", + "nodeType" + ] + }, + "ImportDirective": { + "type": "object", + "additionalProperties": false, + "properties": { + "id": { + "type": "integer" + }, + "src": { + "$ref": "#/definitions/SourceLocation" + }, + "absolutePath": { + "type": "string" + }, + "file": { + "type": "string" + }, + "nameLocation": { + "type": "string" + }, + "scope": { + "type": "integer" + }, + "sourceUnit": { + "type": "integer" + }, + "symbolAliases": { + "type": "array", + "items": { + "type": "object", + "additionalProperties": false, + "properties": { + "foreign": { + "$ref": "#/definitions/Identifier" + }, + "local": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "nameLocation": { + "type": "string" + } + }, + "required": [ + "foreign" + ] + } + }, + "unitAlias": { + "type": "string" + }, + "nodeType": { + "enum": [ + "ImportDirective" + ] + } + }, + "required": [ + "id", + "src", + "absolutePath", + "file", + "scope", + "sourceUnit", + "symbolAliases", + "unitAlias", + "nodeType" + ] + }, + "IndexAccess": { + "type": "object", + "additionalProperties": false, + "properties": { + "id": { + "type": "integer" + }, + "src": { + "$ref": "#/definitions/SourceLocation" + }, + "argumentTypes": { + "anyOf": [ + { + "type": "array", + "items": { + "$ref": "#/definitions/TypeDescriptions" + } + }, + { + "type": "null" + } + ] + }, + "isConstant": { + "type": "boolean" + }, + "isLValue": { + "type": "boolean" + }, + "isPure": { + "type": "boolean" + }, + "lValueRequested": { + "type": "boolean" + }, + "typeDescriptions": { + "$ref": "#/definitions/TypeDescriptions" + }, + "baseExpression": { + "$ref": "#/definitions/Expression" + }, + "indexExpression": { + "anyOf": [ + { + "$ref": "#/definitions/Expression" + }, + { + "type": "null" + } + ] + }, + "nodeType": { + "enum": [ + "IndexAccess" + ] + } + }, + "required": [ + "id", + "src", + "isConstant", + "isLValue", + "isPure", + "lValueRequested", + "typeDescriptions", + "baseExpression", + "nodeType" + ] + }, + "IndexRangeAccess": { + "type": "object", + "additionalProperties": false, + "properties": { + "id": { + "type": "integer" + }, + "src": { + "$ref": "#/definitions/SourceLocation" + }, + "argumentTypes": { + "anyOf": [ + { + "type": "array", + "items": { + "$ref": "#/definitions/TypeDescriptions" + } + }, + { + "type": "null" + } + ] + }, + "isConstant": { + "type": "boolean" + }, + "isLValue": { + "type": "boolean" + }, + "isPure": { + "type": "boolean" + }, + "lValueRequested": { + "type": "boolean" + }, + "typeDescriptions": { + "$ref": "#/definitions/TypeDescriptions" + }, + "baseExpression": { + "$ref": "#/definitions/Expression" + }, + "endExpression": { + "anyOf": [ + { + "$ref": "#/definitions/Expression" + }, + { + "type": "null" + } + ] + }, + "startExpression": { + "anyOf": [ + { + "$ref": "#/definitions/Expression" + }, + { + "type": "null" + } + ] + }, + "nodeType": { + "enum": [ + "IndexRangeAccess" + ] + } + }, + "required": [ + "id", + "src", + "isConstant", + "isLValue", + "isPure", + "lValueRequested", + "typeDescriptions", + "baseExpression", + "nodeType" + ] + }, + "InheritanceSpecifier": { + "type": "object", + "additionalProperties": false, + "properties": { + "id": { + "type": "integer" + }, + "src": { + "$ref": "#/definitions/SourceLocation" + }, + "arguments": { + "anyOf": [ + { + "type": "array", + "items": { + "$ref": "#/definitions/Expression" + } + }, + { + "type": "null" + } + ] + }, + "baseName": { + "anyOf": [ + { + "$ref": "#/definitions/UserDefinedTypeName" + }, + { + "$ref": "#/definitions/IdentifierPath" + } + ] + }, + "nodeType": { + "enum": [ + "InheritanceSpecifier" + ] + } + }, + "required": [ + "id", + "src", + "baseName", + "nodeType" + ] + }, + "InlineAssembly": { + "type": "object", + "additionalProperties": false, + "properties": { + "id": { + "type": "integer" + }, + "src": { + "$ref": "#/definitions/SourceLocation" + }, + "documentation": { + "type": "string" + }, + "AST": { + "$ref": "#/definitions/YulBlock" + }, + "evmVersion": { + "enum": [ + "homestead", + "tangerineWhistle", + "spuriousDragon", + "byzantium", + "constantinople", + "petersburg", + "istanbul", + "berlin", + "london", + "paris", + "shanghai", + "cancun", + "prague", + "osaka" + ] + }, + "externalReferences": { + "type": "array", + "items": { + "type": "object", + "additionalProperties": false, + "properties": { + "declaration": { + "type": "integer" + }, + "isOffset": { + "type": "boolean" + }, + "isSlot": { + "type": "boolean" + }, + "src": { + "$ref": "#/definitions/SourceLocation" + }, + "valueSize": { + "type": "integer" + }, + "suffix": { + "enum": [ + "slot", + "offset", + "length" + ] + } + }, + "required": [ + "declaration", + "isOffset", + "isSlot", + "src", + "valueSize" + ] + } + }, + "flags": { + "type": "array", + "items": { + "enum": [ + "memory-safe" + ] + } + }, + "nodeType": { + "enum": [ + "InlineAssembly" + ] + } + }, + "required": [ + "id", + "src", + "AST", + "evmVersion", + "externalReferences", + "nodeType" + ] + }, + "Literal": { + "type": "object", + "additionalProperties": false, + "properties": { + "id": { + "type": "integer" + }, + "src": { + "$ref": "#/definitions/SourceLocation" + }, + "argumentTypes": { + "anyOf": [ + { + "type": "array", + "items": { + "$ref": "#/definitions/TypeDescriptions" + } + }, + { + "type": "null" + } + ] + }, + "isConstant": { + "type": "boolean" + }, + "isLValue": { + "type": "boolean" + }, + "isPure": { + "type": "boolean" + }, + "lValueRequested": { + "type": "boolean" + }, + "typeDescriptions": { + "$ref": "#/definitions/TypeDescriptions" + }, + "hexValue": { + "type": "string", + "pattern": "^[0-9a-f]*$" + }, + "kind": { + "enum": [ + "bool", + "number", + "string", + "hexString", + "unicodeString" + ] + }, + "subdenomination": { + "anyOf": [ + { + "enum": [ + "seconds", + "minutes", + "hours", + "days", + "weeks", + "wei", + "gwei", + "ether", + "finney", + "szabo" + ] + }, + { + "type": "null" + } + ] + }, + "value": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "nodeType": { + "enum": [ + "Literal" + ] + } + }, + "required": [ + "id", + "src", + "isConstant", + "isLValue", + "isPure", + "lValueRequested", + "typeDescriptions", + "hexValue", + "kind", + "nodeType" + ] + }, + "Mapping": { + "type": "object", + "additionalProperties": false, + "properties": { + "id": { + "type": "integer" + }, + "src": { + "$ref": "#/definitions/SourceLocation" + }, + "typeDescriptions": { + "$ref": "#/definitions/TypeDescriptions" + }, + "keyType": { + "$ref": "#/definitions/TypeName" + }, + "valueType": { + "$ref": "#/definitions/TypeName" + }, + "keyName": { + "type": "string" + }, + "keyNameLocation": { + "type": "string" + }, + "valueName": { + "type": "string" + }, + "valueNameLocation": { + "type": "string" + }, + "nodeType": { + "enum": [ + "Mapping" + ] + } + }, + "required": [ + "id", + "src", + "typeDescriptions", + "keyType", + "valueType", + "nodeType" + ] + }, + "MemberAccess": { + "type": "object", + "additionalProperties": false, + "properties": { + "id": { + "type": "integer" + }, + "src": { + "$ref": "#/definitions/SourceLocation" + }, + "argumentTypes": { + "anyOf": [ + { + "type": "array", + "items": { + "$ref": "#/definitions/TypeDescriptions" + } + }, + { + "type": "null" + } + ] + }, + "isConstant": { + "type": "boolean" + }, + "isLValue": { + "type": "boolean" + }, + "isPure": { + "type": "boolean" + }, + "lValueRequested": { + "type": "boolean" + }, + "typeDescriptions": { + "$ref": "#/definitions/TypeDescriptions" + }, + "expression": { + "$ref": "#/definitions/Expression" + }, + "memberName": { + "type": "string" + }, + "memberLocation": { + "type": "string" + }, + "referencedDeclaration": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ] + }, + "nodeType": { + "enum": [ + "MemberAccess" + ] + } + }, + "required": [ + "id", + "src", + "isConstant", + "isLValue", + "isPure", + "lValueRequested", + "typeDescriptions", + "expression", + "memberName", + "nodeType" + ] + }, + "ModifierDefinition": { + "type": "object", + "additionalProperties": false, + "properties": { + "id": { + "type": "integer" + }, + "src": { + "$ref": "#/definitions/SourceLocation" + }, + "name": { + "type": "string" + }, + "nameLocation": { + "type": "string" + }, + "baseModifiers": { + "anyOf": [ + { + "type": "array", + "items": { + "type": "integer" + } + }, + { + "type": "null" + } + ] + }, + "body": { + "anyOf": [ + { + "$ref": "#/definitions/Block" + }, + { + "type": "null" + } + ] + }, + "documentation": { + "anyOf": [ + { + "$ref": "#/definitions/StructuredDocumentation" + }, + { + "type": "null" + } + ] + }, + "overrides": { + "anyOf": [ + { + "$ref": "#/definitions/OverrideSpecifier" + }, + { + "type": "null" + } + ] + }, + "parameters": { + "$ref": "#/definitions/ParameterList" + }, + "virtual": { + "type": "boolean" + }, + "visibility": { + "$ref": "#/definitions/Visibility" + }, + "nodeType": { + "enum": [ + "ModifierDefinition" + ] + } + }, + "required": [ + "id", + "src", + "name", + "parameters", + "virtual", + "visibility", + "nodeType" + ] + }, + "ModifierInvocation": { + "type": "object", + "additionalProperties": false, + "properties": { + "id": { + "type": "integer" + }, + "src": { + "$ref": "#/definitions/SourceLocation" + }, + "arguments": { + "anyOf": [ + { + "type": "array", + "items": { + "$ref": "#/definitions/Expression" + } + }, + { + "type": "null" + } + ] + }, + "kind": { + "enum": [ + "modifierInvocation", + "baseConstructorSpecifier" + ] + }, + "modifierName": { + "anyOf": [ + { + "$ref": "#/definitions/Identifier" + }, + { + "$ref": "#/definitions/IdentifierPath" + } + ] + }, + "nodeType": { + "enum": [ + "ModifierInvocation" + ] + } + }, + "required": [ + "id", + "src", + "modifierName", + "nodeType" + ] + }, + "NewExpression": { + "type": "object", + "additionalProperties": false, + "properties": { + "id": { + "type": "integer" + }, + "src": { + "$ref": "#/definitions/SourceLocation" + }, + "argumentTypes": { + "anyOf": [ + { + "type": "array", + "items": { + "$ref": "#/definitions/TypeDescriptions" + } + }, + { + "type": "null" + } + ] + }, + "isConstant": { + "type": "boolean" + }, + "isLValue": { + "type": "boolean" + }, + "isPure": { + "type": "boolean" + }, + "lValueRequested": { + "type": "boolean" + }, + "typeDescriptions": { + "$ref": "#/definitions/TypeDescriptions" + }, + "typeName": { + "$ref": "#/definitions/TypeName" + }, + "nodeType": { + "enum": [ + "NewExpression" + ] + } + }, + "required": [ + "id", + "src", + "isConstant", + "isPure", + "lValueRequested", + "typeDescriptions", + "typeName", + "nodeType" + ] + }, + "OverrideSpecifier": { + "type": "object", + "additionalProperties": false, + "properties": { + "id": { + "type": "integer" + }, + "src": { + "$ref": "#/definitions/SourceLocation" + }, + "overrides": { + "anyOf": [ + { + "type": "array", + "items": { + "$ref": "#/definitions/UserDefinedTypeName" + } + }, + { + "type": "array", + "items": { + "$ref": "#/definitions/IdentifierPath" + } + } + ] + }, + "nodeType": { + "enum": [ + "OverrideSpecifier" + ] + } + }, + "required": [ + "id", + "src", + "overrides", + "nodeType" + ] + }, + "ParameterList": { + "type": "object", + "additionalProperties": false, + "properties": { + "id": { + "type": "integer" + }, + "src": { + "$ref": "#/definitions/SourceLocation" + }, + "parameters": { + "type": "array", + "items": { + "$ref": "#/definitions/VariableDeclaration" + } + }, + "nodeType": { + "enum": [ + "ParameterList" + ] + } + }, + "required": [ + "id", + "src", + "parameters", + "nodeType" + ] + }, + "PlaceholderStatement": { + "type": "object", + "additionalProperties": false, + "properties": { + "id": { + "type": "integer" + }, + "src": { + "$ref": "#/definitions/SourceLocation" + }, + "documentation": { + "type": "string" + }, + "nodeType": { + "enum": [ + "PlaceholderStatement" + ] + } + }, + "required": [ + "id", + "src", + "nodeType" + ] + }, + "PragmaDirective": { + "type": "object", + "additionalProperties": false, + "properties": { + "id": { + "type": "integer" + }, + "src": { + "$ref": "#/definitions/SourceLocation" + }, + "literals": { + "type": "array", + "items": { + "type": "string" + } + }, + "nodeType": { + "enum": [ + "PragmaDirective" + ] + } + }, + "required": [ + "id", + "src", + "literals", + "nodeType" + ] + }, + "Return": { + "type": "object", + "additionalProperties": false, + "properties": { + "id": { + "type": "integer" + }, + "src": { + "$ref": "#/definitions/SourceLocation" + }, + "documentation": { + "type": "string" + }, + "expression": { + "anyOf": [ + { + "$ref": "#/definitions/Expression" + }, + { + "type": "null" + } + ] + }, + "functionReturnParameters": { + "type": "integer" + }, + "nodeType": { + "enum": [ + "Return" + ] + } + }, + "required": [ + "id", + "src", + "functionReturnParameters", + "nodeType" + ] + }, + "RevertStatement": { + "type": "object", + "additionalProperties": false, + "properties": { + "id": { + "type": "integer" + }, + "src": { + "$ref": "#/definitions/SourceLocation" + }, + "documentation": { + "type": "string" + }, + "errorCall": { + "$ref": "#/definitions/FunctionCall" + }, + "nodeType": { + "enum": [ + "RevertStatement" + ] + } + }, + "required": [ + "id", + "src", + "errorCall", + "nodeType" + ] + }, + "StructDefinition": { + "type": "object", + "additionalProperties": false, + "properties": { + "id": { + "type": "integer" + }, + "src": { + "$ref": "#/definitions/SourceLocation" + }, + "name": { + "type": "string" + }, + "nameLocation": { + "type": "string" + }, + "canonicalName": { + "type": "string" + }, + "members": { + "type": "array", + "items": { + "$ref": "#/definitions/VariableDeclaration" + } + }, + "scope": { + "type": "integer" + }, + "visibility": { + "$ref": "#/definitions/Visibility" + }, + "documentation": { + "anyOf": [ + { + "$ref": "#/definitions/StructuredDocumentation" + }, + { + "type": "null" + } + ] + }, + "nodeType": { + "enum": [ + "StructDefinition" + ] + } + }, + "required": [ + "id", + "src", + "name", + "canonicalName", + "members", + "scope", + "visibility", + "nodeType" + ] + }, + "StructuredDocumentation": { + "type": "object", + "additionalProperties": false, + "properties": { + "id": { + "type": "integer" + }, + "src": { + "$ref": "#/definitions/SourceLocation" + }, + "text": { + "type": "string" + }, + "nodeType": { + "enum": [ + "StructuredDocumentation" + ] + } + }, + "required": [ + "id", + "src", + "text", + "nodeType" + ] + }, + "TryCatchClause": { + "type": "object", + "additionalProperties": false, + "properties": { + "id": { + "type": "integer" + }, + "src": { + "$ref": "#/definitions/SourceLocation" + }, + "block": { + "$ref": "#/definitions/Block" + }, + "errorName": { + "type": "string" + }, + "parameters": { + "anyOf": [ + { + "$ref": "#/definitions/ParameterList" + }, + { + "type": "null" + } + ] + }, + "nodeType": { + "enum": [ + "TryCatchClause" + ] + } + }, + "required": [ + "id", + "src", + "block", + "errorName", + "nodeType" + ] + }, + "TryStatement": { + "type": "object", + "additionalProperties": false, + "properties": { + "id": { + "type": "integer" + }, + "src": { + "$ref": "#/definitions/SourceLocation" + }, + "documentation": { + "type": "string" + }, + "clauses": { + "type": "array", + "items": { + "$ref": "#/definitions/TryCatchClause" + } + }, + "externalCall": { + "$ref": "#/definitions/FunctionCall" + }, + "nodeType": { + "enum": [ + "TryStatement" + ] + } + }, + "required": [ + "id", + "src", + "clauses", + "externalCall", + "nodeType" + ] + }, + "TupleExpression": { + "type": "object", + "additionalProperties": false, + "properties": { + "id": { + "type": "integer" + }, + "src": { + "$ref": "#/definitions/SourceLocation" + }, + "argumentTypes": { + "anyOf": [ + { + "type": "array", + "items": { + "$ref": "#/definitions/TypeDescriptions" + } + }, + { + "type": "null" + } + ] + }, + "isConstant": { + "type": "boolean" + }, + "isLValue": { + "type": "boolean" + }, + "isPure": { + "type": "boolean" + }, + "lValueRequested": { + "type": "boolean" + }, + "typeDescriptions": { + "$ref": "#/definitions/TypeDescriptions" + }, + "components": { + "type": "array", + "items": { + "anyOf": [ + { + "$ref": "#/definitions/Expression" + }, + { + "type": "null" + } + ] + } + }, + "isInlineArray": { + "type": "boolean" + }, + "nodeType": { + "enum": [ + "TupleExpression" + ] + } + }, + "required": [ + "id", + "src", + "isConstant", + "isLValue", + "isPure", + "lValueRequested", + "typeDescriptions", + "components", + "isInlineArray", + "nodeType" + ] + }, + "UnaryOperation": { + "type": "object", + "additionalProperties": false, + "properties": { + "id": { + "type": "integer" + }, + "src": { + "$ref": "#/definitions/SourceLocation" + }, + "argumentTypes": { + "anyOf": [ + { + "type": "array", + "items": { + "$ref": "#/definitions/TypeDescriptions" + } + }, + { + "type": "null" + } + ] + }, + "isConstant": { + "type": "boolean" + }, + "isLValue": { + "type": "boolean" + }, + "isPure": { + "type": "boolean" + }, + "lValueRequested": { + "type": "boolean" + }, + "typeDescriptions": { + "$ref": "#/definitions/TypeDescriptions" + }, + "operator": { + "enum": [ + "++", + "--", + "-", + "!", + "delete", + "~" + ] + }, + "prefix": { + "type": "boolean" + }, + "subExpression": { + "$ref": "#/definitions/Expression" + }, + "function": { + "type": "integer" + }, + "nodeType": { + "enum": [ + "UnaryOperation" + ] + } + }, + "required": [ + "id", + "src", + "isConstant", + "isLValue", + "isPure", + "lValueRequested", + "typeDescriptions", + "operator", + "prefix", + "subExpression", + "nodeType" + ] + }, + "UncheckedBlock": { + "type": "object", + "additionalProperties": false, + "properties": { + "id": { + "type": "integer" + }, + "src": { + "$ref": "#/definitions/SourceLocation" + }, + "documentation": { + "type": "string" + }, + "statements": { + "type": "array", + "items": { + "$ref": "#/definitions/Statement" + } + }, + "nodeType": { + "enum": [ + "UncheckedBlock" + ] + } + }, + "required": [ + "id", + "src", + "statements", + "nodeType" + ] + }, + "UserDefinedTypeName": { + "type": "object", + "additionalProperties": false, + "properties": { + "id": { + "type": "integer" + }, + "src": { + "$ref": "#/definitions/SourceLocation" + }, + "typeDescriptions": { + "$ref": "#/definitions/TypeDescriptions" + }, + "contractScope": { + "type": "null" + }, + "name": { + "type": "string" + }, + "pathNode": { + "$ref": "#/definitions/IdentifierPath" + }, + "referencedDeclaration": { + "type": "integer" + }, + "nodeType": { + "enum": [ + "UserDefinedTypeName" + ] + } + }, + "required": [ + "id", + "src", + "typeDescriptions", + "referencedDeclaration", + "nodeType" + ] + }, + "UserDefinedValueTypeDefinition": { + "type": "object", + "additionalProperties": false, + "properties": { + "id": { + "type": "integer" + }, + "src": { + "$ref": "#/definitions/SourceLocation" + }, + "name": { + "type": "string" + }, + "nameLocation": { + "type": "string" + }, + "canonicalName": { + "type": "string" + }, + "underlyingType": { + "$ref": "#/definitions/TypeName" + }, + "nodeType": { + "enum": [ + "UserDefinedValueTypeDefinition" + ] + } + }, + "required": [ + "id", + "src", + "name", + "underlyingType", + "nodeType" + ] + }, + "UsingForDirective": { + "type": "object", + "additionalProperties": false, + "properties": { + "id": { + "type": "integer" + }, + "src": { + "$ref": "#/definitions/SourceLocation" + }, + "functionList": { + "type": "array", + "items": { + "anyOf": [ + { + "type": "object", + "additionalProperties": false, + "properties": { + "function": { + "$ref": "#/definitions/IdentifierPath" + } + }, + "required": [ + "function" + ] + }, + { + "type": "object", + "additionalProperties": false, + "properties": { + "operator": { + "enum": [ + "&", + "|", + "^", + "~", + "+", + "-", + "*", + "/", + "%", + "==", + "!=", + "<", + "<=", + ">", + ">=" + ] + }, + "definition": { + "$ref": "#/definitions/IdentifierPath" + } + }, + "required": [ + "operator", + "definition" + ] + } + ] + } + }, + "global": { + "type": "boolean" + }, + "libraryName": { + "anyOf": [ + { + "$ref": "#/definitions/UserDefinedTypeName" + }, + { + "$ref": "#/definitions/IdentifierPath" + } + ] + }, + "typeName": { + "anyOf": [ + { + "$ref": "#/definitions/TypeName" + }, + { + "type": "null" + } + ] + }, + "nodeType": { + "enum": [ + "UsingForDirective" + ] + } + }, + "required": [ + "id", + "src", + "nodeType" + ] + }, + "VariableDeclaration": { + "type": "object", + "additionalProperties": false, + "properties": { + "id": { + "type": "integer" + }, + "src": { + "$ref": "#/definitions/SourceLocation" + }, + "name": { + "type": "string" + }, + "nameLocation": { + "type": "string" + }, + "baseFunctions": { + "anyOf": [ + { + "type": "array", + "items": { + "type": "integer" + } + }, + { + "type": "null" + } + ] + }, + "constant": { + "type": "boolean" + }, + "documentation": { + "anyOf": [ + { + "$ref": "#/definitions/StructuredDocumentation" + }, + { + "type": "null" + } + ] + }, + "functionSelector": { + "type": "string" + }, + "indexed": { + "type": "boolean" + }, + "mutability": { + "$ref": "#/definitions/Mutability" + }, + "overrides": { + "anyOf": [ + { + "$ref": "#/definitions/OverrideSpecifier" + }, + { + "type": "null" + } + ] + }, + "scope": { + "type": "integer" + }, + "stateVariable": { + "type": "boolean" + }, + "storageLocation": { + "$ref": "#/definitions/StorageLocation" + }, + "typeDescriptions": { + "$ref": "#/definitions/TypeDescriptions" + }, + "typeName": { + "anyOf": [ + { + "$ref": "#/definitions/TypeName" + }, + { + "type": "null" + } + ] + }, + "value": { + "anyOf": [ + { + "$ref": "#/definitions/Expression" + }, + { + "type": "null" + } + ] + }, + "visibility": { + "$ref": "#/definitions/Visibility" + }, + "nodeType": { + "enum": [ + "VariableDeclaration" + ] + } + }, + "required": [ + "id", + "src", + "name", + "constant", + "mutability", + "scope", + "stateVariable", + "storageLocation", + "typeDescriptions", + "visibility", + "nodeType" + ] + }, + "VariableDeclarationStatement": { + "type": "object", + "additionalProperties": false, + "properties": { + "id": { + "type": "integer" + }, + "src": { + "$ref": "#/definitions/SourceLocation" + }, + "documentation": { + "type": "string" + }, + "assignments": { + "type": "array", + "items": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ] + } + }, + "declarations": { + "type": "array", + "items": { + "anyOf": [ + { + "$ref": "#/definitions/VariableDeclaration" + }, + { + "type": "null" + } + ] + } + }, + "initialValue": { + "anyOf": [ + { + "$ref": "#/definitions/Expression" + }, + { + "type": "null" + } + ] + }, + "nodeType": { + "enum": [ + "VariableDeclarationStatement" + ] + } + }, + "required": [ + "id", + "src", + "assignments", + "declarations", + "nodeType" + ] + }, + "WhileStatement": { + "type": "object", + "additionalProperties": false, + "properties": { + "id": { + "type": "integer" + }, + "src": { + "$ref": "#/definitions/SourceLocation" + }, + "documentation": { + "type": "string" + }, + "body": { + "anyOf": [ + { + "$ref": "#/definitions/Block" + }, + { + "$ref": "#/definitions/Statement" + } + ] + }, + "condition": { + "$ref": "#/definitions/Expression" + }, + "nodeType": { + "enum": [ + "WhileStatement" + ] + } + }, + "required": [ + "id", + "src", + "body", + "condition", + "nodeType" + ] + }, + "YulStatement": { + "anyOf": [ + { + "$ref": "#/definitions/YulAssignment" + }, + { + "$ref": "#/definitions/YulBlock" + }, + { + "$ref": "#/definitions/YulBreak" + }, + { + "$ref": "#/definitions/YulContinue" + }, + { + "$ref": "#/definitions/YulExpressionStatement" + }, + { + "$ref": "#/definitions/YulLeave" + }, + { + "$ref": "#/definitions/YulForLoop" + }, + { + "$ref": "#/definitions/YulFunctionDefinition" + }, + { + "$ref": "#/definitions/YulIf" + }, + { + "$ref": "#/definitions/YulSwitch" + }, + { + "$ref": "#/definitions/YulVariableDeclaration" + } + ] + }, + "YulExpression": { + "anyOf": [ + { + "$ref": "#/definitions/YulFunctionCall" + }, + { + "$ref": "#/definitions/YulIdentifier" + }, + { + "$ref": "#/definitions/YulLiteral" + } + ] + }, + "YulLiteral": { + "anyOf": [ + { + "$ref": "#/definitions/YulLiteralValue" + }, + { + "$ref": "#/definitions/YulLiteralHexValue" + } + ] + }, + "YulLiteralValue": { + "type": "object", + "additionalProperties": false, + "properties": { + "src": { + "$ref": "#/definitions/SourceLocation" + }, + "value": { + "type": "string" + }, + "kind": { + "enum": [ + "number", + "string", + "bool" + ] + }, + "type": { + "type": "string" + }, + "nodeType": { + "enum": [ + "YulLiteral" + ] + }, + "nativeSrc": { + "$ref": "#/definitions/SourceLocation" + } + }, + "required": [ + "src", + "value", + "kind", + "type", + "nodeType" + ] + }, + "YulLiteralHexValue": { + "type": "object", + "additionalProperties": false, + "properties": { + "src": { + "$ref": "#/definitions/SourceLocation" + }, + "hexValue": { + "type": "string" + }, + "kind": { + "enum": [ + "number", + "string", + "bool" + ] + }, + "type": { + "type": "string" + }, + "value": { + "type": "string" + }, + "nodeType": { + "enum": [ + "YulLiteral" + ] + }, + "nativeSrc": { + "$ref": "#/definitions/SourceLocation" + } + }, + "required": [ + "src", + "hexValue", + "kind", + "type", + "nodeType" + ] + }, + "YulAssignment": { + "type": "object", + "additionalProperties": false, + "properties": { + "src": { + "$ref": "#/definitions/SourceLocation" + }, + "value": { + "$ref": "#/definitions/YulExpression" + }, + "variableNames": { + "type": "array", + "items": { + "$ref": "#/definitions/YulIdentifier" + } + }, + "nodeType": { + "enum": [ + "YulAssignment" + ] + }, + "nativeSrc": { + "$ref": "#/definitions/SourceLocation" + } + }, + "required": [ + "src", + "value", + "variableNames", + "nodeType" + ] + }, + "YulBlock": { + "type": "object", + "additionalProperties": false, + "properties": { + "src": { + "$ref": "#/definitions/SourceLocation" + }, + "statements": { + "type": "array", + "items": { + "$ref": "#/definitions/YulStatement" + } + }, + "nodeType": { + "enum": [ + "YulBlock" + ] + }, + "nativeSrc": { + "$ref": "#/definitions/SourceLocation" + } + }, + "required": [ + "src", + "statements", + "nodeType" + ] + }, + "YulBreak": { + "type": "object", + "additionalProperties": false, + "properties": { + "src": { + "$ref": "#/definitions/SourceLocation" + }, + "nodeType": { + "enum": [ + "YulBreak" + ] + }, + "nativeSrc": { + "$ref": "#/definitions/SourceLocation" + } + }, + "required": [ + "src", + "nodeType" + ] + }, + "YulCase": { + "type": "object", + "additionalProperties": false, + "properties": { + "src": { + "$ref": "#/definitions/SourceLocation" + }, + "body": { + "$ref": "#/definitions/YulBlock" + }, + "value": { + "anyOf": [ + { + "enum": [ + "default" + ] + }, + { + "$ref": "#/definitions/YulLiteral" + } + ] + }, + "nodeType": { + "enum": [ + "YulCase" + ] + }, + "nativeSrc": { + "$ref": "#/definitions/SourceLocation" + } + }, + "required": [ + "src", + "body", + "value", + "nodeType" + ] + }, + "YulContinue": { + "type": "object", + "additionalProperties": false, + "properties": { + "src": { + "$ref": "#/definitions/SourceLocation" + }, + "nodeType": { + "enum": [ + "YulContinue" + ] + }, + "nativeSrc": { + "$ref": "#/definitions/SourceLocation" + } + }, + "required": [ + "src", + "nodeType" + ] + }, + "YulExpressionStatement": { + "type": "object", + "additionalProperties": false, + "properties": { + "src": { + "$ref": "#/definitions/SourceLocation" + }, + "expression": { + "$ref": "#/definitions/YulExpression" + }, + "nodeType": { + "enum": [ + "YulExpressionStatement" + ] + }, + "nativeSrc": { + "$ref": "#/definitions/SourceLocation" + } + }, + "required": [ + "src", + "expression", + "nodeType" + ] + }, + "YulFunctionCall": { + "type": "object", + "additionalProperties": false, + "properties": { + "src": { + "$ref": "#/definitions/SourceLocation" + }, + "arguments": { + "type": "array", + "items": { + "$ref": "#/definitions/YulExpression" + } + }, + "functionName": { + "$ref": "#/definitions/YulIdentifier" + }, + "nodeType": { + "enum": [ + "YulFunctionCall" + ] + }, + "nativeSrc": { + "$ref": "#/definitions/SourceLocation" + } + }, + "required": [ + "src", + "arguments", + "functionName", + "nodeType" + ] + }, + "YulForLoop": { + "type": "object", + "additionalProperties": false, + "properties": { + "src": { + "$ref": "#/definitions/SourceLocation" + }, + "body": { + "$ref": "#/definitions/YulBlock" + }, + "condition": { + "$ref": "#/definitions/YulExpression" + }, + "post": { + "$ref": "#/definitions/YulBlock" + }, + "pre": { + "$ref": "#/definitions/YulBlock" + }, + "nodeType": { + "enum": [ + "YulForLoop" + ] + }, + "nativeSrc": { + "$ref": "#/definitions/SourceLocation" + } + }, + "required": [ + "src", + "body", + "condition", + "post", + "pre", + "nodeType" + ] + }, + "YulFunctionDefinition": { + "type": "object", + "additionalProperties": false, + "properties": { + "src": { + "$ref": "#/definitions/SourceLocation" + }, + "body": { + "$ref": "#/definitions/YulBlock" + }, + "name": { + "type": "string" + }, + "parameters": { + "type": "array", + "items": { + "$ref": "#/definitions/YulTypedName" + } + }, + "returnVariables": { + "type": "array", + "items": { + "$ref": "#/definitions/YulTypedName" + } + }, + "nodeType": { + "enum": [ + "YulFunctionDefinition" + ] + }, + "nativeSrc": { + "$ref": "#/definitions/SourceLocation" + } + }, + "required": [ + "src", + "body", + "name", + "nodeType" + ] + }, + "YulIdentifier": { + "type": "object", + "additionalProperties": false, + "properties": { + "src": { + "$ref": "#/definitions/SourceLocation" + }, + "name": { + "type": "string" + }, + "nodeType": { + "enum": [ + "YulIdentifier" + ] + }, + "nativeSrc": { + "$ref": "#/definitions/SourceLocation" + } + }, + "required": [ + "src", + "name", + "nodeType" + ] + }, + "YulIf": { + "type": "object", + "additionalProperties": false, + "properties": { + "src": { + "$ref": "#/definitions/SourceLocation" + }, + "body": { + "$ref": "#/definitions/YulBlock" + }, + "condition": { + "$ref": "#/definitions/YulExpression" + }, + "nodeType": { + "enum": [ + "YulIf" + ] + }, + "nativeSrc": { + "$ref": "#/definitions/SourceLocation" + } + }, + "required": [ + "src", + "body", + "condition", + "nodeType" + ] + }, + "YulLeave": { + "type": "object", + "additionalProperties": false, + "properties": { + "src": { + "$ref": "#/definitions/SourceLocation" + }, + "nodeType": { + "enum": [ + "YulLeave" + ] + }, + "nativeSrc": { + "$ref": "#/definitions/SourceLocation" + } + }, + "required": [ + "src", + "nodeType" + ] + }, + "YulSwitch": { + "type": "object", + "additionalProperties": false, + "properties": { + "src": { + "$ref": "#/definitions/SourceLocation" + }, + "cases": { + "type": "array", + "items": { + "$ref": "#/definitions/YulCase" + } + }, + "expression": { + "$ref": "#/definitions/YulExpression" + }, + "nodeType": { + "enum": [ + "YulSwitch" + ] + }, + "nativeSrc": { + "$ref": "#/definitions/SourceLocation" + } + }, + "required": [ + "src", + "cases", + "expression", + "nodeType" + ] + }, + "YulTypedName": { + "type": "object", + "additionalProperties": false, + "properties": { + "src": { + "$ref": "#/definitions/SourceLocation" + }, + "name": { + "type": "string" + }, + "type": { + "type": "string" + }, + "nodeType": { + "enum": [ + "YulTypedName" + ] + }, + "nativeSrc": { + "$ref": "#/definitions/SourceLocation" + } + }, + "required": [ + "src", + "name", + "type", + "nodeType" + ] + }, + "YulVariableDeclaration": { + "type": "object", + "additionalProperties": false, + "properties": { + "src": { + "$ref": "#/definitions/SourceLocation" + }, + "value": { + "anyOf": [ + { + "$ref": "#/definitions/YulExpression" + }, + { + "type": "null" + } + ] + }, + "variables": { + "type": "array", + "items": { + "$ref": "#/definitions/YulTypedName" + } + }, + "nodeType": { + "enum": [ + "YulVariableDeclaration" + ] + }, + "nativeSrc": { + "$ref": "#/definitions/SourceLocation" + } + }, + "required": [ + "src", + "variables", + "nodeType" + ] + } + } +} \ No newline at end of file diff --git a/certora_autosetup/solidity_ast/statements.py b/certora_autosetup/solidity_ast/statements.py new file mode 100644 index 00000000..c3096de5 --- /dev/null +++ b/certora_autosetup/solidity_ast/statements.py @@ -0,0 +1,167 @@ +"""Statement nodes of the Solidity compact AST (members of the Statement union).""" + +from typing import TYPE_CHECKING, Literal + +from pydantic import BaseModel, ConfigDict + +from .base import SolcNode + +if TYPE_CHECKING: + from .declarations import ParameterList, VariableDeclaration + from .expressions import FunctionCall + from .unions import Expression, Statement + from .yul import YulBlock + + +class Block(SolcNode): + """A curly-braced statement block.""" + + documentation: str | None = None + statements: "list[Statement] | None" = None + nodeType: Literal["Block"] + + +class Break(SolcNode): + documentation: str | None = None + nodeType: Literal["Break"] + + +class Continue(SolcNode): + documentation: str | None = None + nodeType: Literal["Continue"] + + +class DoWhileStatement(SolcNode): + documentation: str | None = None + body: "Block | Statement" + condition: "Expression" + nodeType: Literal["DoWhileStatement"] + + +class EmitStatement(SolcNode): + documentation: str | None = None + eventCall: "FunctionCall" + nodeType: Literal["EmitStatement"] + + +class ExpressionStatement(SolcNode): + documentation: str | None = None + expression: "Expression" + nodeType: Literal["ExpressionStatement"] + + +class ForStatement(SolcNode): + documentation: str | None = None + body: "Block | Statement" + condition: "Expression | None" = None + initializationExpression: "ExpressionStatement | VariableDeclarationStatement | None" = None + loopExpression: ExpressionStatement | None = None + isSimpleCounterLoop: bool | None = None + nodeType: Literal["ForStatement"] + + +class IfStatement(SolcNode): + documentation: str | None = None + condition: "Expression" + falseBody: "Statement | Block | None" = None + trueBody: "Statement | Block" + nodeType: Literal["IfStatement"] + + +class ExternalReference(BaseModel): + """An entry of ``InlineAssembly.externalReferences``: a Yul identifier that refers + to a Solidity declaration.""" + + model_config = ConfigDict(extra="allow") + + declaration: int + isOffset: bool + isSlot: bool + src: str + valueSize: int + suffix: Literal["slot", "offset", "length"] | None = None + + +class InlineAssembly(SolcNode): + """An ``assembly { ... }`` block; its Yul body lives under the ``AST`` field. + + Dumps from solc <= 0.5 use a different dialect: no ``AST``/``evmVersion``, the + assembly source text in ``operations``, and ``externalReferences`` items keyed + by identifier name (LENIENT_REQUIRED / DELIBERATELY_OPEN deviations). + """ + + documentation: str | None = None + AST: "YulBlock | None" = None + # The schema enumerates the EVM fork names, but each new fork would make every + # assembly-containing source fail whole-file validation until the vendored + # schema catches up — deliberately open (allowlisted in the conformance test). + evmVersion: str | None = None + externalReferences: list[ExternalReference | dict[str, ExternalReference]] + # Same reasoning: new assembly flags arrive with new solc releases. + flags: list[str] | None = None + operations: str | None = None + nodeType: Literal["InlineAssembly"] + + +class PlaceholderStatement(SolcNode): + """The ``_;`` placeholder inside a modifier body.""" + + documentation: str | None = None + nodeType: Literal["PlaceholderStatement"] + + +class Return(SolcNode): + documentation: str | None = None + expression: "Expression | None" = None + # Schema-required, but solc omits it for `return;` inside a modifier body (no + # function to return to) — seen in the wild on solc 0.8.x (conformance + # deviation LENIENT_REQUIRED). + functionReturnParameters: int | None = None + nodeType: Literal["Return"] + + +class RevertStatement(SolcNode): + """A ``revert SomeError(...)`` statement (solc >= 0.8.4).""" + + documentation: str | None = None + errorCall: "FunctionCall" + nodeType: Literal["RevertStatement"] + + +class TryStatement(SolcNode): + documentation: str | None = None + clauses: "list[TryCatchClause]" + externalCall: "FunctionCall" + nodeType: Literal["TryStatement"] + + +class TryCatchClause(SolcNode): + """A ``try``-success or ``catch`` clause of a TryStatement.""" + + block: Block + errorName: str + parameters: "ParameterList | None" = None + nodeType: Literal["TryCatchClause"] + + +class UncheckedBlock(SolcNode): + """An ``unchecked { ... }`` block (solc >= 0.8.0).""" + + documentation: str | None = None + statements: "list[Statement]" + nodeType: Literal["UncheckedBlock"] + + +class VariableDeclarationStatement(SolcNode): + documentation: str | None = None + assignments: list[int | None] + declarations: "list[VariableDeclaration | None]" + initialValue: "Expression | None" = None + nodeType: Literal["VariableDeclarationStatement"] + + +class WhileStatement(SolcNode): + documentation: str | None = None + body: "Block | Statement" + condition: "Expression" + nodeType: Literal["WhileStatement"] diff --git a/certora_autosetup/solidity_ast/traversal.py b/certora_autosetup/solidity_ast/traversal.py new file mode 100644 index 00000000..60ab69cd --- /dev/null +++ b/certora_autosetup/solidity_ast/traversal.py @@ -0,0 +1,117 @@ +"""Traversal utilities over typed AST nodes, plus the legacy raw parent-graph builder.""" + +from typing import Any, Iterable, Iterator, TypeVar, cast + +from pydantic import BaseModel + +from .base import AstNode, SolcNode + +N = TypeVar("N", bound=AstNode) + + +def iter_children(node: AstNode) -> Iterator[AstNode]: + """Direct AST children of a node, in model-field declaration order. + + Helper models that are not themselves AST nodes (e.g. import symbol aliases, + ``using``-directive function lists) are transparent containers: AST nodes found + inside them are yielded as direct children of ``node``. Extra fields captured by + ``model_extra`` (unknown to the model set) are not descended into. + """ + for name in type(node).model_fields: + yield from _child_nodes(getattr(node, name)) + + +def _child_nodes(value: Any) -> Iterator[AstNode]: + if isinstance(value, AstNode): + yield value + elif isinstance(value, BaseModel): + for name in type(value).model_fields: + yield from _child_nodes(getattr(value, name)) + elif isinstance(value, list): + for item in value: + yield from _child_nodes(item) + + +def walk(node: AstNode) -> Iterator[AstNode]: + """Pre-order DFS over ``node`` and all its descendants (iterative — deep + expression chains cannot hit the interpreter recursion limit).""" + stack = [node] + while stack: + current = stack.pop() + yield current + stack.extend(reversed(list(iter_children(current)))) + + +def find_all(root: AstNode, node_type: type[N] | tuple[type[N], ...]) -> Iterator[N]: + """All nodes of the given type(s) in the subtree rooted at ``root`` (inclusive), + in document order. The typed replacement for ``nodeType == "X"`` scans.""" + for node in walk(root): + if isinstance(node, node_type): + yield node + + +def build_node_index(root: AstNode) -> dict[int, AstNode]: + """Map every id-carrying node in the subtree to its instance (Yul nodes carry + no id and are not indexed).""" + return {node.id: node for node in walk(root) if isinstance(node, SolcNode)} + + +def build_parent_map(root: AstNode) -> dict[int, int]: + """Map child node id -> parent node id over the subtree, for id-carrying nodes. + + Nodes nested inside transparent helper containers are attached to the nearest + id-carrying AST ancestor. + """ + parent_map: dict[int, int] = {} + stack: list[tuple[AstNode, int | None]] = [(root, None)] + while stack: + node, parent_id = stack.pop() + node_id = node.id if isinstance(node, SolcNode) else None + if node_id is not None and parent_id is not None: + parent_map[node_id] = parent_id + enclosing = node_id if node_id is not None else parent_id + stack.extend((child, enclosing) for child in iter_children(node)) + return parent_map + + +def build_parent_graph_json( + raw_asts: dict[str, Any] | Iterable[tuple[str, Any]], +) -> dict[str, dict[str, dict[str, str]]]: + """Parent graph over RAW ``.asts.json`` data (a full dict, or streamed + ``(relative_path, path_data)`` pairs from ``loader.stream_raw_units``), in the + exact legacy format written to ``all_ast_parent_graph.json``: + {rel_path: {abs_path: {child_id: parent_id}}} with string ids. + + Deliberately operates on the raw dicts with the historical child heuristic (a child + is any direct dict value, or list element, carrying an ``id`` key) and preserves + raw key order, so ``json.dump(..., indent=2)`` output stays byte-identical to what + existing readers of the file expect. Use :func:`build_parent_map` for typed code. + """ + units: Iterable[tuple[str, Any]] + if isinstance(raw_asts, dict): + units = cast("Iterable[tuple[str, Any]]", raw_asts.items()) + else: + units = raw_asts + parent_graph: dict[str, dict[str, dict[str, str]]] = {} + for relative_path, path_data in units: + parent_graph[relative_path] = {} + for absolute_path, nodes in path_data.items(): + parent_graph[relative_path][absolute_path] = {} + for node_id, node in nodes.items(): + if not isinstance(node, dict): + continue + for child_id in _legacy_child_ids(node): + parent_graph[relative_path][absolute_path][str(child_id)] = str(node_id) + return parent_graph + + +def _legacy_child_ids(node: dict[str, Any]) -> list[Any]: + child_ids = [] + for value in node.values(): + if isinstance(value, dict) and "id" in value: + child_ids.append(value["id"]) + elif isinstance(value, list): + for item in value: + if isinstance(item, dict) and "id" in item: + child_ids.append(item["id"]) + return child_ids diff --git a/certora_autosetup/solidity_ast/types.py b/certora_autosetup/solidity_ast/types.py new file mode 100644 index 00000000..25e35700 --- /dev/null +++ b/certora_autosetup/solidity_ast/types.py @@ -0,0 +1,71 @@ +"""Type-name nodes of the Solidity compact AST (members of the TypeName union).""" + +from typing import TYPE_CHECKING, Literal + +from .base import SolcNode, StateMutability, TypeDescriptions, Visibility + +if TYPE_CHECKING: + from .declarations import ParameterList + from .unions import Expression, TypeName + + +class ArrayTypeName(SolcNode): + """A static or dynamic array type, e.g. ``uint256[]`` or ``bytes32[4]``.""" + + typeDescriptions: TypeDescriptions + baseType: "TypeName" + length: "Expression | None" = None + nodeType: Literal["ArrayTypeName"] + + +class ElementaryTypeName(SolcNode): + """A built-in type name, e.g. ``uint256``, ``address``, ``bytes``.""" + + typeDescriptions: TypeDescriptions + name: str + stateMutability: StateMutability | None = None + nodeType: Literal["ElementaryTypeName"] + + +class FunctionTypeName(SolcNode): + """A function type, e.g. ``function (uint) external returns (bool)``.""" + + typeDescriptions: TypeDescriptions + parameterTypes: "ParameterList" + returnParameterTypes: "ParameterList" + stateMutability: StateMutability + visibility: Visibility + nodeType: Literal["FunctionTypeName"] + + +class Mapping(SolcNode): + """A mapping type, e.g. ``mapping(address owner => uint256 balance)``.""" + + typeDescriptions: TypeDescriptions + keyType: "TypeName" + valueType: "TypeName" + keyName: str | None = None + keyNameLocation: str | None = None + valueName: str | None = None + valueNameLocation: str | None = None + nodeType: Literal["Mapping"] + + +class IdentifierPath(SolcNode): + """A (possibly dotted) path referring to a declaration, e.g. ``Lib.Struct``.""" + + name: str + nameLocations: list[str] | None = None + referencedDeclaration: int + nodeType: Literal["IdentifierPath"] + + +class UserDefinedTypeName(SolcNode): + """A reference to a user-defined type (struct, enum, contract, UDVT).""" + + typeDescriptions: TypeDescriptions + contractScope: None = None + name: str | None = None + pathNode: IdentifierPath | None = None + referencedDeclaration: int + nodeType: Literal["UserDefinedTypeName"] diff --git a/certora_autosetup/solidity_ast/unions.py b/certora_autosetup/solidity_ast/unions.py new file mode 100644 index 00000000..2de14019 --- /dev/null +++ b/certora_autosetup/solidity_ast/unions.py @@ -0,0 +1,405 @@ +"""Discriminated unions over the AST node models, the schema-name registry, and the +forward-reference rebuild wiring. + +Import this module (or the package) before validating any node model: importing it +resolves every cross-module forward reference and rebuilds all models. Each union +carries an UnknownNode fallback member selected for any unrecognized ``nodeType``, +so ASTs from solc versions newer than the vendored schema degrade per-node instead +of failing whole-file validation. +""" + +from typing import Annotated, Union + +from pydantic import Discriminator, Tag + +from . import declarations, expressions, statements, types, yul +from .base import UNKNOWN_TAG, AstNode, UnknownNode, tag_by_node_type +from .yul import YulExpression, YulLiteral, YulStatement + + +from .types import ( + ArrayTypeName, + ElementaryTypeName, + FunctionTypeName, + IdentifierPath, + Mapping, + UserDefinedTypeName, +) +from .expressions import ( + Assignment, + BinaryOperation, + Conditional, + ElementaryTypeNameExpression, + FunctionCall, + FunctionCallOptions, + Identifier, + IndexAccess, + IndexRangeAccess, + Literal, + MemberAccess, + NewExpression, + TupleExpression, + UnaryOperation, +) +from .statements import ( + Block, + Break, + Continue, + DoWhileStatement, + EmitStatement, + ExpressionStatement, + ForStatement, + IfStatement, + InlineAssembly, + PlaceholderStatement, + Return, + RevertStatement, + TryCatchClause, + TryStatement, + UncheckedBlock, + VariableDeclarationStatement, + WhileStatement, +) +from .declarations import ( + ContractDefinition, + EnumDefinition, + EnumValue, + ErrorDefinition, + EventDefinition, + FunctionDefinition, + ImportDirective, + InheritanceSpecifier, + ModifierDefinition, + ModifierInvocation, + OverrideSpecifier, + ParameterList, + PragmaDirective, + SourceUnit, + StorageLayoutSpecifier, + StructDefinition, + StructuredDocumentation, + UserDefinedValueTypeDefinition, + UsingForDirective, + VariableDeclaration, +) +from .yul import ( + YulAssignment, + YulBlock, + YulBreak, + YulCase, + YulContinue, + YulExpressionStatement, + YulForLoop, + YulFunctionCall, + YulFunctionDefinition, + YulIdentifier, + YulIf, + YulLeave, + YulLiteralHexValue, + YulLiteralValue, + YulSwitch, + YulTypedName, + YulVariableDeclaration, +) + +_EXPRESSION_TAGS = frozenset({"Assignment", "BinaryOperation", "Conditional", "ElementaryTypeNameExpression", "FunctionCall", "FunctionCallOptions", "Identifier", "IndexAccess", "IndexRangeAccess", "Literal", "MemberAccess", "NewExpression", "TupleExpression", "UnaryOperation"}) + +Expression = Annotated[ + Union[ + Annotated[Assignment, Tag("Assignment")], + Annotated[BinaryOperation, Tag("BinaryOperation")], + Annotated[Conditional, Tag("Conditional")], + Annotated[ElementaryTypeNameExpression, Tag("ElementaryTypeNameExpression")], + Annotated[FunctionCall, Tag("FunctionCall")], + Annotated[FunctionCallOptions, Tag("FunctionCallOptions")], + Annotated[Identifier, Tag("Identifier")], + Annotated[IndexAccess, Tag("IndexAccess")], + Annotated[IndexRangeAccess, Tag("IndexRangeAccess")], + Annotated[Literal, Tag("Literal")], + Annotated[MemberAccess, Tag("MemberAccess")], + Annotated[NewExpression, Tag("NewExpression")], + Annotated[TupleExpression, Tag("TupleExpression")], + Annotated[UnaryOperation, Tag("UnaryOperation")], + Annotated[UnknownNode, Tag(UNKNOWN_TAG)], + ], + Discriminator(tag_by_node_type(_EXPRESSION_TAGS)), +] +"""Any Solidity expression node (or UnknownNode).""" + +_STATEMENT_TAGS = frozenset({"Block", "Break", "Continue", "DoWhileStatement", "EmitStatement", "ExpressionStatement", "ForStatement", "IfStatement", "InlineAssembly", "PlaceholderStatement", "Return", "RevertStatement", "TryStatement", "UncheckedBlock", "VariableDeclarationStatement", "WhileStatement"}) + +Statement = Annotated[ + Union[ + Annotated[Block, Tag("Block")], + Annotated[Break, Tag("Break")], + Annotated[Continue, Tag("Continue")], + Annotated[DoWhileStatement, Tag("DoWhileStatement")], + Annotated[EmitStatement, Tag("EmitStatement")], + Annotated[ExpressionStatement, Tag("ExpressionStatement")], + Annotated[ForStatement, Tag("ForStatement")], + Annotated[IfStatement, Tag("IfStatement")], + Annotated[InlineAssembly, Tag("InlineAssembly")], + Annotated[PlaceholderStatement, Tag("PlaceholderStatement")], + Annotated[Return, Tag("Return")], + Annotated[RevertStatement, Tag("RevertStatement")], + Annotated[TryStatement, Tag("TryStatement")], + Annotated[UncheckedBlock, Tag("UncheckedBlock")], + Annotated[VariableDeclarationStatement, Tag("VariableDeclarationStatement")], + Annotated[WhileStatement, Tag("WhileStatement")], + Annotated[UnknownNode, Tag(UNKNOWN_TAG)], + ], + Discriminator(tag_by_node_type(_STATEMENT_TAGS)), +] +"""Any Solidity statement node (or UnknownNode).""" + +_TYPENAME_TAGS = frozenset({"ArrayTypeName", "ElementaryTypeName", "FunctionTypeName", "Mapping", "UserDefinedTypeName"}) + +TypeName = Annotated[ + Union[ + Annotated[ArrayTypeName, Tag("ArrayTypeName")], + Annotated[ElementaryTypeName, Tag("ElementaryTypeName")], + Annotated[FunctionTypeName, Tag("FunctionTypeName")], + Annotated[Mapping, Tag("Mapping")], + Annotated[UserDefinedTypeName, Tag("UserDefinedTypeName")], + Annotated[UnknownNode, Tag(UNKNOWN_TAG)], + ], + Discriminator(tag_by_node_type(_TYPENAME_TAGS)), +] +"""Any type-name node (or UnknownNode).""" + +# EventDefinition is absent from the vendored schema's SourceUnit.nodes union, but +# solc >= 0.8.22 allows file-level events (seen in the wild; conformance deviation +# DELIBERATELY_OPEN on SourceUnit.nodes). +_SOURCEUNITNODE_TAGS = frozenset({"ContractDefinition", "EnumDefinition", "ErrorDefinition", "EventDefinition", "FunctionDefinition", "ImportDirective", "PragmaDirective", "StructDefinition", "UserDefinedValueTypeDefinition", "UsingForDirective", "VariableDeclaration"}) + +SourceUnitNode = Annotated[ + Union[ + Annotated[ContractDefinition, Tag("ContractDefinition")], + Annotated[EnumDefinition, Tag("EnumDefinition")], + Annotated[ErrorDefinition, Tag("ErrorDefinition")], + Annotated[EventDefinition, Tag("EventDefinition")], + Annotated[FunctionDefinition, Tag("FunctionDefinition")], + Annotated[ImportDirective, Tag("ImportDirective")], + Annotated[PragmaDirective, Tag("PragmaDirective")], + Annotated[StructDefinition, Tag("StructDefinition")], + Annotated[UserDefinedValueTypeDefinition, Tag("UserDefinedValueTypeDefinition")], + Annotated[UsingForDirective, Tag("UsingForDirective")], + Annotated[VariableDeclaration, Tag("VariableDeclaration")], + Annotated[UnknownNode, Tag(UNKNOWN_TAG)], + ], + Discriminator(tag_by_node_type(_SOURCEUNITNODE_TAGS)), +] +"""Any node that may appear directly in SourceUnit.nodes (or UnknownNode).""" + +_CONTRACTBODYNODE_TAGS = frozenset({"EnumDefinition", "ErrorDefinition", "EventDefinition", "FunctionDefinition", "ModifierDefinition", "StructDefinition", "UserDefinedValueTypeDefinition", "UsingForDirective", "VariableDeclaration"}) + +ContractBodyNode = Annotated[ + Union[ + Annotated[EnumDefinition, Tag("EnumDefinition")], + Annotated[ErrorDefinition, Tag("ErrorDefinition")], + Annotated[EventDefinition, Tag("EventDefinition")], + Annotated[FunctionDefinition, Tag("FunctionDefinition")], + Annotated[ModifierDefinition, Tag("ModifierDefinition")], + Annotated[StructDefinition, Tag("StructDefinition")], + Annotated[UserDefinedValueTypeDefinition, Tag("UserDefinedValueTypeDefinition")], + Annotated[UsingForDirective, Tag("UsingForDirective")], + Annotated[VariableDeclaration, Tag("VariableDeclaration")], + Annotated[UnknownNode, Tag(UNKNOWN_TAG)], + ], + Discriminator(tag_by_node_type(_CONTRACTBODYNODE_TAGS)), +] +"""Any node that may appear directly in ContractDefinition.nodes (or UnknownNode).""" + +_NODE_TAGS = frozenset({"ArrayTypeName", "Assignment", "BinaryOperation", "Block", "Break", "Conditional", "Continue", "ContractDefinition", "DoWhileStatement", "ElementaryTypeName", "ElementaryTypeNameExpression", "EmitStatement", "EnumDefinition", "EnumValue", "ErrorDefinition", "EventDefinition", "ExpressionStatement", "ForStatement", "FunctionCall", "FunctionCallOptions", "FunctionDefinition", "FunctionTypeName", "Identifier", "IdentifierPath", "IfStatement", "ImportDirective", "IndexAccess", "IndexRangeAccess", "InheritanceSpecifier", "InlineAssembly", "Literal", "Mapping", "MemberAccess", "ModifierDefinition", "ModifierInvocation", "NewExpression", "OverrideSpecifier", "ParameterList", "PlaceholderStatement", "PragmaDirective", "Return", "RevertStatement", "SourceUnit", "StorageLayoutSpecifier", "StructDefinition", "StructuredDocumentation", "TryCatchClause", "TryStatement", "TupleExpression", "UnaryOperation", "UncheckedBlock", "UserDefinedTypeName", "UserDefinedValueTypeDefinition", "UsingForDirective", "VariableDeclaration", "VariableDeclarationStatement", "WhileStatement", "YulAssignment", "YulBlock", "YulBreak", "YulCase", "YulContinue", "YulExpressionStatement", "YulForLoop", "YulFunctionCall", "YulFunctionDefinition", "YulIdentifier", "YulIf", "YulLeave", "YulLiteral", "YulSwitch", "YulTypedName", "YulVariableDeclaration"}) + +Node = Annotated[ + Union[ + Annotated[ArrayTypeName, Tag("ArrayTypeName")], + Annotated[Assignment, Tag("Assignment")], + Annotated[BinaryOperation, Tag("BinaryOperation")], + Annotated[Block, Tag("Block")], + Annotated[Break, Tag("Break")], + Annotated[Conditional, Tag("Conditional")], + Annotated[Continue, Tag("Continue")], + Annotated[ContractDefinition, Tag("ContractDefinition")], + Annotated[DoWhileStatement, Tag("DoWhileStatement")], + Annotated[ElementaryTypeName, Tag("ElementaryTypeName")], + Annotated[ElementaryTypeNameExpression, Tag("ElementaryTypeNameExpression")], + Annotated[EmitStatement, Tag("EmitStatement")], + Annotated[EnumDefinition, Tag("EnumDefinition")], + Annotated[EnumValue, Tag("EnumValue")], + Annotated[ErrorDefinition, Tag("ErrorDefinition")], + Annotated[EventDefinition, Tag("EventDefinition")], + Annotated[ExpressionStatement, Tag("ExpressionStatement")], + Annotated[ForStatement, Tag("ForStatement")], + Annotated[FunctionCall, Tag("FunctionCall")], + Annotated[FunctionCallOptions, Tag("FunctionCallOptions")], + Annotated[FunctionDefinition, Tag("FunctionDefinition")], + Annotated[FunctionTypeName, Tag("FunctionTypeName")], + Annotated[Identifier, Tag("Identifier")], + Annotated[IdentifierPath, Tag("IdentifierPath")], + Annotated[IfStatement, Tag("IfStatement")], + Annotated[ImportDirective, Tag("ImportDirective")], + Annotated[IndexAccess, Tag("IndexAccess")], + Annotated[IndexRangeAccess, Tag("IndexRangeAccess")], + Annotated[InheritanceSpecifier, Tag("InheritanceSpecifier")], + Annotated[InlineAssembly, Tag("InlineAssembly")], + Annotated[Literal, Tag("Literal")], + Annotated[Mapping, Tag("Mapping")], + Annotated[MemberAccess, Tag("MemberAccess")], + Annotated[ModifierDefinition, Tag("ModifierDefinition")], + Annotated[ModifierInvocation, Tag("ModifierInvocation")], + Annotated[NewExpression, Tag("NewExpression")], + Annotated[OverrideSpecifier, Tag("OverrideSpecifier")], + Annotated[ParameterList, Tag("ParameterList")], + Annotated[PlaceholderStatement, Tag("PlaceholderStatement")], + Annotated[PragmaDirective, Tag("PragmaDirective")], + Annotated[Return, Tag("Return")], + Annotated[RevertStatement, Tag("RevertStatement")], + Annotated[SourceUnit, Tag("SourceUnit")], + Annotated[StorageLayoutSpecifier, Tag("StorageLayoutSpecifier")], + Annotated[StructDefinition, Tag("StructDefinition")], + Annotated[StructuredDocumentation, Tag("StructuredDocumentation")], + Annotated[TryCatchClause, Tag("TryCatchClause")], + Annotated[TryStatement, Tag("TryStatement")], + Annotated[TupleExpression, Tag("TupleExpression")], + Annotated[UnaryOperation, Tag("UnaryOperation")], + Annotated[UncheckedBlock, Tag("UncheckedBlock")], + Annotated[UserDefinedTypeName, Tag("UserDefinedTypeName")], + Annotated[UserDefinedValueTypeDefinition, Tag("UserDefinedValueTypeDefinition")], + Annotated[UsingForDirective, Tag("UsingForDirective")], + Annotated[VariableDeclaration, Tag("VariableDeclaration")], + Annotated[VariableDeclarationStatement, Tag("VariableDeclarationStatement")], + Annotated[WhileStatement, Tag("WhileStatement")], + Annotated[YulAssignment, Tag("YulAssignment")], + Annotated[YulBlock, Tag("YulBlock")], + Annotated[YulBreak, Tag("YulBreak")], + Annotated[YulCase, Tag("YulCase")], + Annotated[YulContinue, Tag("YulContinue")], + Annotated[YulExpressionStatement, Tag("YulExpressionStatement")], + Annotated[YulForLoop, Tag("YulForLoop")], + Annotated[YulFunctionCall, Tag("YulFunctionCall")], + Annotated[YulFunctionDefinition, Tag("YulFunctionDefinition")], + Annotated[YulIdentifier, Tag("YulIdentifier")], + Annotated[YulIf, Tag("YulIf")], + Annotated[YulLeave, Tag("YulLeave")], + Annotated[YulLiteralValue | YulLiteralHexValue, Tag("YulLiteral")], + Annotated[YulSwitch, Tag("YulSwitch")], + Annotated[YulTypedName, Tag("YulTypedName")], + Annotated[YulVariableDeclaration, Tag("YulVariableDeclaration")], + Annotated[UnknownNode, Tag(UNKNOWN_TAG)], + ], + Discriminator(tag_by_node_type(_NODE_TAGS)), +] +"""Any concrete AST node of any kind (or UnknownNode).""" + +# Schema definition name -> model class ("SourceUnit" is the schema root, the two +# YulLiteral* variants share nodeType "YulLiteral"). +MODEL_BY_SCHEMA_DEF: dict[str, type[AstNode]] = { + "ArrayTypeName": types.ArrayTypeName, + "Assignment": expressions.Assignment, + "BinaryOperation": expressions.BinaryOperation, + "Block": statements.Block, + "Break": statements.Break, + "Conditional": expressions.Conditional, + "Continue": statements.Continue, + "ContractDefinition": declarations.ContractDefinition, + "DoWhileStatement": statements.DoWhileStatement, + "ElementaryTypeName": types.ElementaryTypeName, + "ElementaryTypeNameExpression": expressions.ElementaryTypeNameExpression, + "EmitStatement": statements.EmitStatement, + "EnumDefinition": declarations.EnumDefinition, + "EnumValue": declarations.EnumValue, + "ErrorDefinition": declarations.ErrorDefinition, + "EventDefinition": declarations.EventDefinition, + "ExpressionStatement": statements.ExpressionStatement, + "ForStatement": statements.ForStatement, + "FunctionCall": expressions.FunctionCall, + "FunctionCallOptions": expressions.FunctionCallOptions, + "FunctionDefinition": declarations.FunctionDefinition, + "FunctionTypeName": types.FunctionTypeName, + "Identifier": expressions.Identifier, + "IdentifierPath": types.IdentifierPath, + "IfStatement": statements.IfStatement, + "ImportDirective": declarations.ImportDirective, + "IndexAccess": expressions.IndexAccess, + "IndexRangeAccess": expressions.IndexRangeAccess, + "InheritanceSpecifier": declarations.InheritanceSpecifier, + "InlineAssembly": statements.InlineAssembly, + "Literal": expressions.Literal, + "Mapping": types.Mapping, + "MemberAccess": expressions.MemberAccess, + "ModifierDefinition": declarations.ModifierDefinition, + "ModifierInvocation": declarations.ModifierInvocation, + "NewExpression": expressions.NewExpression, + "OverrideSpecifier": declarations.OverrideSpecifier, + "ParameterList": declarations.ParameterList, + "PlaceholderStatement": statements.PlaceholderStatement, + "PragmaDirective": declarations.PragmaDirective, + "Return": statements.Return, + "RevertStatement": statements.RevertStatement, + "SourceUnit": declarations.SourceUnit, + "StorageLayoutSpecifier": declarations.StorageLayoutSpecifier, + "StructDefinition": declarations.StructDefinition, + "StructuredDocumentation": declarations.StructuredDocumentation, + "TryCatchClause": statements.TryCatchClause, + "TryStatement": statements.TryStatement, + "TupleExpression": expressions.TupleExpression, + "UnaryOperation": expressions.UnaryOperation, + "UncheckedBlock": statements.UncheckedBlock, + "UserDefinedTypeName": types.UserDefinedTypeName, + "UserDefinedValueTypeDefinition": declarations.UserDefinedValueTypeDefinition, + "UsingForDirective": declarations.UsingForDirective, + "VariableDeclaration": declarations.VariableDeclaration, + "VariableDeclarationStatement": statements.VariableDeclarationStatement, + "WhileStatement": statements.WhileStatement, + "YulAssignment": yul.YulAssignment, + "YulBlock": yul.YulBlock, + "YulBreak": yul.YulBreak, + "YulCase": yul.YulCase, + "YulContinue": yul.YulContinue, + "YulExpressionStatement": yul.YulExpressionStatement, + "YulForLoop": yul.YulForLoop, + "YulFunctionCall": yul.YulFunctionCall, + "YulFunctionDefinition": yul.YulFunctionDefinition, + "YulIdentifier": yul.YulIdentifier, + "YulIf": yul.YulIf, + "YulLeave": yul.YulLeave, + "YulLiteralHexValue": yul.YulLiteralHexValue, + "YulLiteralValue": yul.YulLiteralValue, + "YulSwitch": yul.YulSwitch, + "YulTypedName": yul.YulTypedName, + "YulVariableDeclaration": yul.YulVariableDeclaration, +} + + +_UNION_ALIASES: dict[str, object] = { + "Expression": Expression, + "Statement": Statement, + "TypeName": TypeName, + "SourceUnitNode": SourceUnitNode, + "ContractBodyNode": ContractBodyNode, + "Node": Node, + "YulStatement": YulStatement, + "YulExpression": YulExpression, + "YulLiteral": YulLiteral, +} + +_NAMESPACE: dict[str, object] = { + **{cls.__name__: cls for cls in MODEL_BY_SCHEMA_DEF.values()}, + **_UNION_ALIASES, +} + +# Node modules reference classes and unions from sibling modules as string forward +# refs only (they import nothing from each other at runtime). Resolve everything by +# injecting the shared namespace into each module's globals — never clobbering a +# name the module already defines (e.g. typing.Literal vs the Literal node class) — +# and rebuild every model once. +for _mod in (types, expressions, statements, declarations, yul): + for _name, _obj in _NAMESPACE.items(): + if not hasattr(_mod, _name): + setattr(_mod, _name, _obj) + +for _cls in {*MODEL_BY_SCHEMA_DEF.values(), UnknownNode}: + _cls.model_rebuild(force=True) + diff --git a/certora_autosetup/solidity_ast/yul.py b/certora_autosetup/solidity_ast/yul.py new file mode 100644 index 00000000..4bd86407 --- /dev/null +++ b/certora_autosetup/solidity_ast/yul.py @@ -0,0 +1,184 @@ +"""Yul AST nodes (the ``AST`` of an ``InlineAssembly`` node, solc >= 0.6). + +Self-contained: unlike the Solidity node modules, the union aliases +(``YulLiteral``/``YulExpression``/``YulStatement``) are defined here and all models +are rebuilt at import time, so this module validates on its own. +""" + +from typing import Annotated, Literal, Union + +from pydantic import Discriminator, Tag + +from .base import UNKNOWN_TAG, UnknownNode, YulNode, tag_by_node_type + + +class YulAssignment(YulNode): + value: "YulExpression" + variableNames: list["YulIdentifier"] + nodeType: Literal["YulAssignment"] + + +class YulBlock(YulNode): + statements: list["YulStatement"] + nodeType: Literal["YulBlock"] + + +class YulBreak(YulNode): + nodeType: Literal["YulBreak"] + + +class YulCase(YulNode): + body: YulBlock + value: "Literal['default'] | YulLiteral" + nodeType: Literal["YulCase"] + + +class YulContinue(YulNode): + nodeType: Literal["YulContinue"] + + +class YulExpressionStatement(YulNode): + expression: "YulExpression" + nodeType: Literal["YulExpressionStatement"] + + +class YulForLoop(YulNode): + body: YulBlock + condition: "YulExpression" + post: YulBlock + pre: YulBlock + nodeType: Literal["YulForLoop"] + + +class YulFunctionCall(YulNode): + arguments: list["YulExpression"] + functionName: "YulIdentifier" + nodeType: Literal["YulFunctionCall"] + + +class YulFunctionDefinition(YulNode): + body: YulBlock + name: str + parameters: "list[YulTypedName] | None" = None + returnVariables: "list[YulTypedName] | None" = None + nodeType: Literal["YulFunctionDefinition"] + + +class YulIdentifier(YulNode): + name: str + nodeType: Literal["YulIdentifier"] + + +class YulIf(YulNode): + body: YulBlock + condition: "YulExpression" + nodeType: Literal["YulIf"] + + +class YulLeave(YulNode): + nodeType: Literal["YulLeave"] + + +class YulLiteralValue(YulNode): + value: str + kind: Literal["number", "string", "bool"] + type: str + nodeType: Literal["YulLiteral"] + + +class YulLiteralHexValue(YulNode): + hexValue: str + kind: Literal["number", "string", "bool"] + type: str + value: str | None = None + nodeType: Literal["YulLiteral"] + + +class YulSwitch(YulNode): + cases: list[YulCase] + expression: "YulExpression" + nodeType: Literal["YulSwitch"] + + +class YulTypedName(YulNode): + name: str + type: str + nodeType: Literal["YulTypedName"] + + +class YulVariableDeclaration(YulNode): + value: "YulExpression | None" = None + variables: list[YulTypedName] + nodeType: Literal["YulVariableDeclaration"] + + +# Union aliases mirroring the schema's helper definitions. Both YulLiteral variants +# share the "YulLiteral" tag, so within that branch pydantic picks by fields. +YulLiteral = YulLiteralValue | YulLiteralHexValue + +YulExpression = Annotated[ + Union[ + Annotated[YulFunctionCall, Tag("YulFunctionCall")], + Annotated[YulIdentifier, Tag("YulIdentifier")], + Annotated[YulLiteralValue | YulLiteralHexValue, Tag("YulLiteral")], + Annotated[UnknownNode, Tag(UNKNOWN_TAG)], + ], + Discriminator( + tag_by_node_type(frozenset({"YulFunctionCall", "YulIdentifier", "YulLiteral"})) + ), +] + +YulStatement = Annotated[ + Union[ + Annotated[YulAssignment, Tag("YulAssignment")], + Annotated[YulBlock, Tag("YulBlock")], + Annotated[YulBreak, Tag("YulBreak")], + Annotated[YulContinue, Tag("YulContinue")], + Annotated[YulExpressionStatement, Tag("YulExpressionStatement")], + Annotated[YulLeave, Tag("YulLeave")], + Annotated[YulForLoop, Tag("YulForLoop")], + Annotated[YulFunctionDefinition, Tag("YulFunctionDefinition")], + Annotated[YulIf, Tag("YulIf")], + Annotated[YulSwitch, Tag("YulSwitch")], + Annotated[YulVariableDeclaration, Tag("YulVariableDeclaration")], + Annotated[UnknownNode, Tag(UNKNOWN_TAG)], + ], + Discriminator( + tag_by_node_type( + frozenset( + { + "YulAssignment", + "YulBlock", + "YulBreak", + "YulContinue", + "YulExpressionStatement", + "YulLeave", + "YulForLoop", + "YulFunctionDefinition", + "YulIf", + "YulSwitch", + "YulVariableDeclaration", + } + ) + ) + ), +] + +# The Yul namespace is fully defined above, so forward refs resolve right here. +YulAssignment.model_rebuild() +YulBlock.model_rebuild() +YulBreak.model_rebuild() +YulCase.model_rebuild() +YulContinue.model_rebuild() +YulExpressionStatement.model_rebuild() +YulForLoop.model_rebuild() +YulFunctionCall.model_rebuild() +YulFunctionDefinition.model_rebuild() +YulIdentifier.model_rebuild() +YulIf.model_rebuild() +YulLeave.model_rebuild() +YulLiteralValue.model_rebuild() +YulLiteralHexValue.model_rebuild() +YulSwitch.model_rebuild() +YulTypedName.model_rebuild() +YulVariableDeclaration.model_rebuild() diff --git a/certora_autosetup/utils/compilation_workarounds.py b/certora_autosetup/utils/compilation_workarounds.py index 55b7da04..14653899 100644 --- a/certora_autosetup/utils/compilation_workarounds.py +++ b/certora_autosetup/utils/compilation_workarounds.py @@ -18,6 +18,13 @@ from certora_autosetup.utils.constants import DEFAULT_SOLC_VERSION, SolcConvention from certora_autosetup.utils.enhanced_config_manager import ConfigManager +from certora_autosetup.utils.import_diagnostics import ( + UnresolvedImport, + classify_unresolved_import, + describe_unresolved_imports, + parse_unresolved_imports, + path_from_source_location_line, +) from certora_autosetup.utils.library_harness import ( LibrarySpec, build_consumer_harness_source, @@ -28,16 +35,70 @@ from certora_autosetup.utils.remappings import build_packages_from_remapping_sources from certora_autosetup.utils.solc_version_resolver import ( extract_pragma_spec, + pragma_admits, + read_pragma_from_source_file, resolve_pragma_to_version, ) from certora_autosetup.utils.types import ContractHandle +# Solc's legacy-codegen stack-too-deep, as opposed to the YulException the via-ir pipeline +# raises. Matched against whitespace-normalized output, since solc hard-wraps diagnostics. +_STACK_TOO_DEEP_RE = re.compile(r"CompilerError:\s*Stack too deep", re.IGNORECASE) + +# How many contracts may be given via-ir one at a time before the rest of the scene is +# switched with them. Each individual escalation costs one full compile of the scene to +# discover, so this bounds the walk; below it, contracts keep their summaries. +VIA_IR_SCENE_THRESHOLD = 10 + class AbstractMainContractError(Exception): """Raised when the main (verify-target) contract compiled to no bytecode — it is abstract (or lacks a constructor) and therefore cannot be verified.""" +class UnimplementedContractError(Exception): + """Raised when a contract in the compilation input leaves a function it inherits + unimplemented, which Solidity accepts only on an ``abstract`` contract.""" + + +class UnsatisfiableSolcPinError(Exception): + """Raised when a contract's pinned solc binary is absent and no installed compiler + satisfies its pragma, so no substitution can make the project compile.""" + + +# The contract name is quoted by solc, so it survives the hard wrap that +# ``_normalize_ws`` folds away; the source location that follows is optional +# because only the diagnostic itself is guaranteed to be in the output. +_UNIMPLEMENTED_CONTRACT_RE = re.compile( + r'Contract "(?P[^"]+)" should be marked as abstract\.' + r"(?: --> (?P[^\s:]+):\d+:\d+)?" +) + + +@dataclass(frozen=True) +class BlockedSolcPin: + """A contract whose pragma no installed compiler can satisfy.""" + + contract_name: str + pragma_spec: str + + +@dataclass(frozen=True) +class SolcFallbackPlan: + """Which installed compiler, if any, stands in for ``failed_solc`` at each + contract pinned to it. + + ``compliant`` maps such a contract to an installed compiler its pragma admits; + ``blocked`` holds the ones no installed compiler can serve. A plan with any + ``blocked`` entry is terminal: substituting a compiler the pragma rejects would + fail identically on the next compile. + """ + + failed_solc: str + compliant: Dict[str, str] + blocked: Tuple[BlockedSolcPin, ...] + + def _normalize_ws(text: str) -> str: """Collapse every run of whitespace (including solc's hard-wrap newlines) to a single space, so multi-substring detectors survive line wrapping. @@ -63,17 +124,38 @@ def _path_from_compiling_line(line: str) -> Optional[str]: return line.removeprefix(prefix).removesuffix(suffix) -# A solc source-location line, e.g. `` --> contracts/Foo.sol:120:9:``. It names the -# offending file in a whole-project (non-autofinder) solc error, where there is no -# ``Compiling ...`` progress line to recover it from. -_SOURCE_LOCATION_RE = re.compile(r"^\s*-->\s+(?P.+?):\d+:\d+:?\s*$") +# The remediation hint solc attaches to a diagnostic whose fix is "compile this with the +# IR pipeline", whatever the diagnostic itself is called: the flag spelling (``--via-ir``), +# the Standard-JSON key (``viaIR: true``), or a mention of the pipeline by name ("via-ir +# pipeline", "IR pipeline"). Keying on the hint rather than on one diagnostic's wording +# covers the whole family. +# +# Written against whitespace-normalized text (see ``_normalize_ws``); ``-\s?`` absorbs a +# solc hard-wrap inside a hyphenated token (``--via-\nir`` normalizes to ``--via- ir``). +# The conf key ``solc_via_ir`` is outside the family — it appears in the "unsupported solc +# version for solc_via_ir" error, which calls for the opposite fix — so every alternative +# demands either the ``--`` flag spelling, the JSON key with its colon, or the word +# "pipeline". ``\bIR`` keeps prose like "through their pipeline" out. +_VIA_IR_HINT_RE = re.compile( + r"--\s?via-\s?ir\b|viaIR:\s?true|(?:\bvia-\s?ir|\bIR)\s+pipeline", + re.IGNORECASE, +) +# Diagnostics that carry the same hint while via-ir is only ONE of the remedies solc +# offers ("... while enabling the optimizer. Otherwise, try removing local variables"), +# so the hint does not mean via-ir is required. Their escalation ladder — optimizer, +# solc's default Yul steps, then relaxing the autofinder assertion — belongs to +# stack_too_deep_via_ir and the yul_exception_* workarounds. +_MULTI_REMEDY_DIAGNOSTIC_RE = re.compile( + r"YulException|Stack\s+too\s+deep|too\s+deep\s+in(?:side)?\s+the\s+stack", + re.IGNORECASE, +) -def _path_from_source_location_line(line: str) -> Optional[str]: - """Return ```` from a solc `` --> :::`` source-location - line, or None if ``line`` isn't one.""" - match = _SOURCE_LOCATION_RE.match(line) - return match.group("path") if match else None +# The label opening a solc diagnostic ("Warning: ...", "TypeError: ...", +# "UnimplementedFeatureError: ...", "YulException: ..."). It delimits one diagnostic from +# the next, so a hint is read together with the diagnostic that owns it — and only with +# that diagnostic's own source locations. +_DIAGNOSTIC_START_RE = re.compile(r"^\s*(?:[A-Za-z]*Error|Warning|Info|Note|YulException)\b") def _find_compiling_path_before(lines: List[str], idx: int, max_lookback: Optional[int] = None) -> Optional[str]: @@ -127,6 +209,7 @@ def __init__( verbose: int = 0, solc_convention: SolcConvention = SolcConvention.CERTORA, build_config_dir: Optional[Path] = None, + declared_via_ir: bool = False, ): self.project_root = project_root # Where foundry.toml / remappings.txt / package.json are read from. Distinct from @@ -135,11 +218,19 @@ def __init__( self.build_config_dir = build_config_dir or project_root self.solc_convention = solc_convention self.verbose = verbose + # Classification of the compilation output the loop returned on, empty when that output + # had no source-not-found error at all (see `_record_import_diagnostics`). Read by + # callers to name the failure class in the terminal error text instead of only the phase + # that failed, so it must describe *that* failure and never an earlier pass's. + self.last_import_diagnostics: List[UnresolvedImport] = [] self._remappings_workaround_applied = False # (consumer, lib) pairs already covered by a generated harness in this run. # Used as a loop guard — if the prover still reports the same pair after we # wrapped the consumer, the workaround stops firing to avoid spinning. self._harnessed_libs: Set[Tuple[str, str]] = set() + # Installed compilers usable as a substitute, probed once (shutil.which + + # subprocess per candidate) and reused across every pass of the loop. + self._solc_candidates: Optional[List[Tuple[str, str]]] = None # consumer -> [(lib_name, lib_path), ...] in insertion order. A second # firing for the same consumer (different missing library) regenerates the # consumer's harness covering every library it has needed so far. @@ -149,6 +240,13 @@ def __init__( # to the scalar on finalize — an unseeded partial map (e.g. a single cancun # entry) must never be promoted to a global scalar. self._evm_version_seeded = False + # Contracts given via-ir one at a time, for the scene-wide threshold in + # _apply_via_ir_workaround. + self._via_ir_contracts: Set[str] = set() + # What the project's own build config asks for. The value is never inherited into + # the conf (build_systems/base.py drops it deliberately), but it does say where a + # via-ir escalation is going to end up, so the walk can be skipped. + self.declared_via_ir = declared_via_ir # Convert default version to the detected convention if solc_convention == SolcConvention.SOLC_SELECT and solc_default_version.startswith("solc") \ and not solc_default_version.startswith("solc-"): @@ -264,6 +362,19 @@ def _detect_abstract_main_contract(self, output: str, compilation_config: Dict) no_bytecode = {m.group(1) for m in re.finditer(r"Contract (\S+) has no bytecode", output)} return main_contract if main_contract in no_bytecode else None + def _detect_unimplemented_contract(self, output: str) -> Optional[Tuple[str, Optional[str]]]: + """Return the (contract name, declaring file) a contract in the input was + rejected for: it inherits functions it does not implement, and is not declared + ``abstract``. + + The file is None when the diagnostic carries no source location. This is a + source-level defect, so the compilation settings have no bearing on it. + """ + match = _UNIMPLEMENTED_CONTRACT_RE.search(_normalize_ws(output)) + if match is None: + return None + return match.group("name"), match.group("path") + # ========================================================================= # Main entry point for running compilation with workarounds # ========================================================================= @@ -328,6 +439,9 @@ def run_compilation_with_workarounds( # same (stale) output. applied_this_pass: Set[str] = set() + # Rebuilt from each failed output, before the workaround table runs. + solc_fallback_plan: Optional[SolcFallbackPlan] = None + # Initialize workarounds list workarounds = [ CompilationWorkaround( @@ -346,7 +460,9 @@ def run_compilation_with_workarounds( ), CompilationWorkaround( name="solc_not_found_fallback", - detect_fn=lambda output: self._detect_solc_not_found(output), + # The plan is built once per pass below, because a plan with no viable + # substitution is terminal and has to be acted on before this table runs. + detect_fn=lambda output: solc_fallback_plan if (solc_fallback_plan and solc_fallback_plan.compliant) else None, apply_fn=self._apply_solc_fallback_workaround, enabled=solc_pinned, ), @@ -362,12 +478,7 @@ def run_compilation_with_workarounds( ), CompilationWorkaround( name="source_not_found_packages", - detect_fn=lambda output: ( - "detected" - if self._has_source_not_found(output) - and not self._remappings_workaround_applied - else None - ), + detect_fn=lambda output: self._detect_source_not_found(output, compilation_config), apply_fn=self._apply_source_not_found_packages_workaround, enabled=True, ), @@ -382,9 +493,56 @@ def run_compilation_with_workarounds( # contract's entry from its pragma is always safe. enabled=True, ), + # solc's own advice on a legacy stack-too-deep is to compile via-ir "while + # enabling the optimizer". The optimizer is what reclaims stack slots, and it + # does so under legacy codegen too. Enabling it alone leaves codegen as it is, + # where via-ir replaces it: via-ir inlines internal functions, which can leave + # the internal summaries CVL applies with nothing to attach to. It does not + # always cost them, but it is a risk worth not taking when the optimizer alone + # may do. + CompilationWorkaround( + name="stack_too_deep_optimizer", + detect_fn=lambda output: ( + "detected" + if self._detect_stack_too_deep_errors(output, contracts) is not None + and self._yul_optimizer_pending(compilation_config, contracts) + else None + ), + apply_fn=self._apply_optimizer, + enabled=not global_via_ir_enabled, + ), + # The optimizer can clear the contract compile and still leave the + # autofinder-instrumented one over the stack limit, since instrumentation adds + # slots of its own. Falling back puts the local-variable finders for those files + # at risk; via-ir puts the internal summaries at risk in every file it is enabled + # for. Neither loss is certain — a file can fall back and still give useful + # finders, and an inlined contract can still be summarized — but the second + # exposure is the wider one, so this rung comes first. Only fires on output + # produced after the optimizer was tried, not in the pass that just enabled it. + CompilationWorkaround( + name="stack_too_deep_autofinder", + detect_fn=lambda output: ( + "detected" + if self._autofinder_relaxation_pending(output, compilation_config) + and not self._yul_optimizer_pending(compilation_config, contracts) + and "stack_too_deep_optimizer" not in applied_this_pass + else None + ), + apply_fn=self._apply_yul_exception_workaround, + enabled=not global_via_ir_enabled, + ), + # Last of the legacy rungs: only once the optimizer is on and the autofinder + # assertion is no longer what is failing the run. CompilationWorkaround( name="stack_too_deep_via_ir", - detect_fn=lambda output: self._detect_stack_too_deep_errors(output, contracts), + detect_fn=lambda output: ( + self._detect_stack_too_deep_errors(output, contracts) + if not self._yul_optimizer_pending(compilation_config, contracts) + and not self._autofinder_relaxation_pending(output, compilation_config) + and "stack_too_deep_optimizer" not in applied_this_pass + and "stack_too_deep_autofinder" not in applied_this_pass + else None + ), apply_fn=self._apply_via_ir_workaround_to_config, enabled=not global_via_ir_enabled, ), @@ -433,7 +591,7 @@ def run_compilation_with_workarounds( and self._yul_optimizer_pending(compilation_config, contracts) else None ), - apply_fn=self._apply_optimizer_for_via_ir, + apply_fn=self._apply_optimizer, enabled=True, ), # Escalation after yul_exception_add_optimizer: with optimizer + @@ -487,6 +645,12 @@ def run_compilation_with_workarounds( detect_fn=lambda output: self._detect_missing_library(output, contracts), apply_fn=self._apply_missing_library_harness_to_config, enabled=True, + # Firing again for the same consumer with a different library + # regenerates the harness source covering every library so far. Its + # _harnessed_libs guard cannot bound that — the guard is keyed on the + # consumer name, which this apply replaces with the harness's, so a + # recurring link error is looked up under a name that was never + # recorded; max_retries is the bound. ), # Catch-all: final attempt before setup_prover falls back to the # import-patch pass. @@ -516,6 +680,12 @@ def run_compilation_with_workarounds( with open(config_file, "w") as f: json.dump(compilation_config, f, indent=2) + # Every state this loop has compiled, starting with the one the first compile + # runs on. Local to the call: a caller may run the loop again on the same + # manager (fixconf does, once either side of the import patch), and the second + # run legitimately revisits what the first one did. + seen_states = {self._retry_state(cmd, compilation_config, updated_config_dict)} + output = "" while retry_count <= max_retries: # Run compilation @@ -523,6 +693,7 @@ def run_compilation_with_workarounds( output = result.stdout + result.stderr if result.returncode == 0: + self._record_import_diagnostics(output, compilation_config, log_summary=False) self._finalize_compile_maps(compilation_config, updated_config_dict, config_file) return True, output, updated_config_dict @@ -538,6 +709,7 @@ def run_compilation_with_workarounds( # otherwise fire first and delay this). abstract_main_contract = self._detect_abstract_main_contract(output, compilation_config) if abstract_main_contract is not None: + self._record_import_diagnostics(output, compilation_config, log_summary=False) self._finalize_compile_maps(compilation_config, updated_config_dict, config_file) raise AbstractMainContractError( f"Main contract '{abstract_main_contract}' compiled to no bytecode: it is abstract " @@ -545,6 +717,42 @@ def run_compilation_with_workarounds( f"verified. Re-run with a concrete implementation as the main contract." ) + # Also terminal: a contract in the input inherits functions it never + # implements, so solc refuses to compile it at all. Only editing that + # contract fixes it; without this the catch-all workaround fires and spends + # further compilations reaching the same error. + unimplemented = self._detect_unimplemented_contract(output) + if unimplemented is not None: + contract_name, declared_in = unimplemented + self._finalize_compile_maps(compilation_config, updated_config_dict, config_file) + location = f" ({declared_in})" if declared_in else "" + raise UnimplementedContractError( + f"Contract '{contract_name}'{location} does not implement every function it " + f"inherits, so Solidity requires it to be marked abstract and refuses to " + f"compile it as written. See the 'Missing implementation' notes in the " + f"compiler output for the functions it still owes." + ) + + # Terminal, non-recoverable case: a contract is pinned to a compiler that + # is not installed and none of the installed ones satisfy its pragma. + # Substituting one anyway reproduces this same failure next pass, so stop + # here and name the compiler that has to be installed. + # Only for a pin the conf actually carries: _seed_compile_maps pins every + # contract to the default compiler, and refusing to proceed over a pin + # autosetup invented itself would fail runs the user never constrained. + solc_fallback_plan = ( + self._plan_solc_fallback(output, updated_config_dict, contracts) if solc_pinned else None + ) + if solc_fallback_plan is not None and solc_fallback_plan.blocked: + self._finalize_compile_maps(compilation_config, updated_config_dict, config_file) + installed = ", ".join(binary for binary, _ in self._solc_fallback_candidates()) or "none" + pins = "; ".join(f"{p.contract_name} requires '{p.pragma_spec}'" for p in solc_fallback_plan.blocked) + raise UnsatisfiableSolcPinError( + f"Compiler '{solc_fallback_plan.failed_solc}' is not installed and no installed " + f"compiler satisfies the pragma of: {pins}. Installed compilers considered: " + f"{installed}. Install '{solc_fallback_plan.failed_solc}' to compile this project." + ) + # One pass over the failed output: apply EVERY applicable workaround # before recompiling — one full certoraRun per pass is expensive, so # a pass fixes as much of this output as it can. detect_fns run @@ -552,7 +760,6 @@ def run_compilation_with_workarounds( # gated on conf/manager state (e.g. _remappings_workaround_applied) # see the pass's own effects. applied_this_pass.clear() - state_before = self._retry_state(cmd, compilation_config, updated_config_dict) def try_workaround(workaround: CompilationWorkaround) -> bool: """Detect and, on a hit, apply — returns whether it applied.""" @@ -596,19 +803,31 @@ def try_workaround(workaround: CompilationWorkaround) -> bool: self.log(output, "WARNING") else: self.log("Compilation failed with no applicable workaround", "ERROR") + self._record_import_diagnostics(output, compilation_config) self._finalize_compile_maps(compilation_config, updated_config_dict, config_file) return False, output, updated_config_dict - # If the whole pass changed nothing, recompiling would reproduce the - # identical failure — stop here instead of burning another certoraRun. - if self._retry_state(cmd, compilation_config, updated_config_dict) == state_before: + # A state this loop has already compiled produces the same failure again, + # whether the pass changed nothing at all or a later workaround undid what + # an earlier one did. Either way retrying cannot converge. + state = self._retry_state(cmd, compilation_config, updated_config_dict) + if state in seen_states: + # Classifying this pass's output turns "the loop is circling" into the reason + # it cannot get out (e.g. the package is installed and the file inside it is + # the one missing). It is logged as part of the ERROR below, not separately. + diagnosis = describe_unresolved_imports( + self._record_import_diagnostics(output, compilation_config, log_summary=False) + ) self.log( - f"Workarounds applied ({', '.join(sorted(applied_this_pass))}) but the conf " - f"and command are unchanged — retrying would fail identically, giving up", + f"Workarounds applied ({', '.join(sorted(applied_this_pass))}) but the conf and " + f"command are back to a state that already failed to compile — retrying would " + f"fail identically, giving up" + + (f"\n{diagnosis}" if diagnosis else ""), "ERROR", ) self._finalize_compile_maps(compilation_config, updated_config_dict, config_file) return False, output, updated_config_dict + seen_states.add(state) retry_count += 1 conf_contents = json.dumps(compilation_config, indent=2) @@ -623,6 +842,7 @@ def try_workaround(workaround: CompilationWorkaround) -> bool: self.log(f"Max retries ({max_retries}) exceeded for workarounds", "ERROR") self.log("Final compilation output:", "ERROR") self.log(output, "ERROR") + self._record_import_diagnostics(output, compilation_config) self._finalize_compile_maps(compilation_config, updated_config_dict, config_file) return False, output, updated_config_dict @@ -630,10 +850,9 @@ def try_workaround(workaround: CompilationWorkaround) -> bool: def _retry_state(cmd: List[str], compilation_config: Dict, updated_config_dict: Dict) -> str: """Serialized snapshot of everything a workaround can change to make the next compilation retry behave differently: the command line and both - config dicts. Used by the no-progress check in - ``run_compilation_with_workarounds`` — a pass that leaves this snapshot - identical was a no-op, so retrying would reproduce the same failure - verbatim. + config dicts. ``run_compilation_with_workarounds`` memoizes it per pass and + stops when one comes round again — the same command over the same conf + reproduces the same failure verbatim, so the loop is circling. Invariant on apply_fns: any application that makes real progress MUST change the command or one of the two conf dicts. Progress expressed @@ -646,6 +865,24 @@ def _retry_state(cmd: List[str], compilation_config: Dict, updated_config_dict: # Detection methods # ========================================================================= + def _autofinder_relaxation_pending(self, output: str, compilation_config: Dict) -> bool: + """True when the run is failing on the autofinder-instrumented compile and the + assertion that turns that into a failure is still on. + + certoraRun compiles each file twice: once as written, once instrumented to expose + internal functions and local variables. The instrumented copy carries extra stack + slots, so it can be the only one over the limit — the contracts themselves compile. + Relaxing the assertion accepts a finder-less fallback for those files while leaving + codegen alone. + """ + if not compilation_config.get("assert_autofinder_success", False): + return False + normalized = re.sub(r"\s+", " ", output) + return ( + "Encountered an exception generating autofinder" in normalized + and bool(_STACK_TOO_DEEP_RE.search(normalized)) + ) + def _detect_stack_too_deep_errors( self, output: str, contracts: List[ContractHandle] ) -> Optional[str]: @@ -702,7 +939,7 @@ def _detect_stack_too_deep_errors( if not line.startswith("CompilerError: Stack too deep"): continue for j in range(i + 1, min(i + 6, len(lines))): - src_path = _path_from_source_location_line(lines[j]) + src_path = path_from_source_location_line(lines[j]) if src_path is None: continue contract_name = self._get_contract_name_from_path(src_path, contracts) @@ -773,21 +1010,45 @@ def _detect_via_ir_required( the affected contract name. Contracts start on plain settings and gain via-ir strictly out of - necessity; this is the necessity signal for non-stack reasons, e.g. - "UnimplementedFeatureError: Require with a custom error is only - available using the via-ir pipeline." Matching is whitespace-normalized - per compiled unit, since solc hard-wraps the phrase. + necessity. solc spells this necessity several ways ("Require with a + custom error is only available using the via-ir pipeline.", "Copying of + type ... to storage is not supported in legacy (only supported by the + IR pipeline)."), so the detector keys on the remediation hint they share + (``_VIA_IR_HINT_RE``) rather than on one diagnostic's wording. + + The hint alone is not the signal: solc appends it to stack-too-deep and + YulException diagnostics too, where via-ir is one remedy among several + and the optimizer/Yul ladder must be climbed first. A hint therefore + counts only inside a diagnostic that offers no other remedy + (``_MULTI_REMEDY_DIAGNOSTIC_RE``). Matching is whitespace-normalized per + diagnostic, since solc hard-wraps the text. """ - marker = "only available using the via-ir pipeline" + lines = output.split("\n") + + def diagnostic_blocks(unit_lines: List[str]) -> List[List[str]]: + """Split solc output into one group of lines per diagnostic, so a hint, + the diagnostic offering it and its source locations stay together and a + neighbouring diagnostic's ``-->`` lines stay out.""" + blocks: List[List[str]] = [[]] + for line in unit_lines: + if _DIAGNOSTIC_START_RE.match(line) or _path_from_compiling_line(line) is not None: + blocks.append([]) + blocks[-1].append(line) + return blocks + + def requires_via_ir(block: List[str]) -> bool: + normalized = _normalize_ws("\n".join(block)) + return bool(_VIA_IR_HINT_RE.search(normalized)) and not _MULTI_REMEDY_DIAGNOSTIC_RE.search(normalized) + current_path: Optional[str] = None segment: List[str] = [] def segment_hit() -> Optional[str]: - if current_path and marker in _normalize_ws("\n".join(segment)): + if current_path and any(requires_via_ir(b) for b in diagnostic_blocks(segment)): return self._get_contract_name_from_path(current_path, contracts) return None - for line in output.split("\n"): + for line in lines: path = _path_from_compiling_line(line) if path is not None and "to expose internal function information" not in line: hit = segment_hit() @@ -801,7 +1062,27 @@ def segment_hit() -> Optional[str]: hit = segment_hit() if hit: self.log(f"Detected via-ir-only feature for {hit} (path: {current_path})") - return hit + return hit + + # Whole-project compile: certoraRun prints no per-file "Compiling ..." + # progress line, so no segment carries a path and the offending file is named + # only in the `-->` source-location lines of the diagnostic itself. Runs last so + # a per-unit hit takes precedence. + for block in diagnostic_blocks(lines): + if not requires_via_ir(block): + continue + for line in block: + src_path = path_from_source_location_line(line) + if src_path is None: + continue + contract_name = self._get_contract_name_from_path(src_path, contracts) + if contract_name: + self.log(f"Detected via-ir-only feature for {contract_name} (path: {src_path})") + return contract_name + self.log(f"Warning: Could not map path '{src_path}' to contract name", "WARNING") + break + + return None def _detect_yul_exception_stack_too_deep(self, output: str) -> bool: """Detect YulException with stack too deep error. @@ -922,6 +1203,60 @@ def _detect_unsupported_solc_via_ir(self, output: str, contracts: List[ContractH return None return None + def _detect_source_not_found(self, output: str, compilation_config: Dict) -> Optional[str]: + """Fire the packages rebuild on a source-not-found output, classifying it on the way. + + The classification is recorded and logged, not used as a gate: the loop's no-progress + check already stops cleanly when a rebuild produces the identical packages list, and + gating here would mean a misread output silently skips a workaround that works. + """ + if not self._has_source_not_found(output) or self._remappings_workaround_applied: + return None + self._classify_unresolved_imports(output, compilation_config) + return "detected" + + def _record_import_diagnostics( + self, output: str, compilation_config: Dict, log_summary: bool = True + ) -> List[UnresolvedImport]: + """Refresh ``last_import_diagnostics`` from ``output`` on the way out of the loop. + + Only an output carrying a source-not-found error can be classified, so every other + outcome must *clear* the field rather than leave it alone: callers append it to their + terminal error text (``setup_prover.run_compilation_analysis``), and a verdict from an + earlier pass — whose import problem this pass may well have fixed — would blame a missing + dependency for a failure that has nothing to do with imports. + """ + if not self._has_source_not_found(output): + self.last_import_diagnostics = [] + return [] + return self._classify_unresolved_imports(output, compilation_config, log_summary) + + def _classify_unresolved_imports( + self, output: str, compilation_config: Dict, log_summary: bool = True + ) -> List[UnresolvedImport]: + """Classify every source-not-found error in ``output`` against the conf's packages, + store the result on the manager, and log the summary (unless the caller prints the + classification itself). + + Purely diagnostic: it changes no conf and gates no workaround, so a change in solc's + output format degrades to today's (less precise) messages rather than aborting the + workaround loop. + """ + try: + packages = compilation_config.get("packages", []) or [] + failures = [ + classify_unresolved_import(source_unit, packages, self.project_root, importer) + for source_unit, importer in parse_unresolved_imports(output) + ] + except Exception as e: + self.log(f"Could not classify unresolved imports: {e}", "WARNING") + return [] + + self.last_import_diagnostics = failures + if failures and log_summary: + self.log(f"Unresolved imports:\n{describe_unresolved_imports(failures)}", "WARNING") + return failures + def _has_remappings_conflict(self, output: str) -> bool: return "package.json and remappings.txt include duplicated keys in" in output @@ -1148,7 +1483,7 @@ def _yul_optimizer_pending( self._optimizer_off(optimize_map.get(c.contract_name)) for c in contracts ) - def _apply_optimizer_for_via_ir( + def _apply_optimizer( self, _detect_result: str, updated_config_dict: Dict, @@ -1156,7 +1491,10 @@ def _apply_optimizer_for_via_ir( config_file: Path, contracts: List[ContractHandle], ) -> Dict: - """Apply optimizer alongside via-ir to resolve YulException stack-too-deep. + """Enable the optimizer to resolve a stack-too-deep, under either codegen pipeline. + + Reached from the legacy rung (CompilerError: Stack too deep) and from the Yul rung + (YulException), because the optimizer's stack-limit evader is what relieves both. With a per-contract solc_optimize_map (foundry compilation_restrictions) only the entries whose optimizer is off are enabled — explicit project @@ -1173,13 +1511,13 @@ def _apply_optimizer_for_via_ir( optimize_map[name] = "200" updated_config_dict["solc_optimize_map"] = optimize_map self.log( - "Detected YulException stack-too-deep with via-ir — enabling the " - f"optimizer (200 runs) in solc_optimize_map for {enabled}", + "Stack-too-deep — enabling the optimizer (200 runs) in " + f"solc_optimize_map for {enabled}", "WARNING", ) else: self.log( - "Detected YulException stack-too-deep with via-ir — adding solc_optimize 200", + "Stack-too-deep — adding solc_optimize 200", "WARNING", ) compilation_config["solc_optimize"] = "200" @@ -1273,51 +1611,86 @@ def _apply_disable_build_cache( def _apply_solc_fallback_workaround( self, - failed_solc: str, + plan: SolcFallbackPlan, updated_config_dict: Dict, compilation_config: Dict, config_file: Path, _contracts: List[ContractHandle], ) -> Dict: - """Fall back from a missing versioned solc binary. - - Checks if plain 'solc' provides the version we need; if not, uses the - default versioned binary (convention-aware, e.g. solc8.34 or solc-0.8.34). - """ - fallback = self._pick_solc_fallback() - self.log(f"Falling back from '{failed_solc}' to '{fallback}'", "WARNING") - + """Substitute a missing versioned solc binary per the plan.""" # solc is seeded into compiler_map up front (see _seed_compile_maps), so - # rewrite the bad binary there — every entry pinned to it — rather than + # replace the bad binary there — every entry pinned to it — rather than # setting the scalar solc, which can't coexist with compiler_map. A # uniform map collapses back to scalar solc in _normalize_compile_maps. # compilation_config shares this map object with updated_config_dict, # so the in-place rewrite is visible in the disk write below too — no mirroring needed. cmap = updated_config_dict.get("compiler_map") assert isinstance(cmap, dict), "compiler_map is seeded before any workaround runs" - for name, version in cmap.items(): - if version == failed_solc: - cmap[name] = fallback + for name, replacement in plan.compliant.items(): + self.log(f"Falling back from '{plan.failed_solc}' to '{replacement}' for {name}", "WARNING") + cmap[name] = replacement with open(config_file, "w") as f: json.dump(compilation_config, f, indent=2) return updated_config_dict - def _pick_solc_fallback(self) -> str: - """Choose the best solc fallback: plain 'solc' if it matches the desired version, else the default.""" - desired = self._extract_version_from_solc_name(self.solc_default_version) - if not desired: - return "solc" + def _solc_fallback_candidates(self) -> List[Tuple[str, str]]: + """Installed compilers that may stand in for a missing binary, as + (binary name, semantic version), best first. - plain_version = self._get_plain_solc_version() - if plain_version and plain_version == desired: - return "solc" + Compilers of unknown version are omitted: the version is what a contract's + pragma is checked against. + """ + if self._solc_candidates is None: + candidates: List[Tuple[str, str]] = [] + # The project's own default comes first: whatever `solc` happens to be on + # PATH is unrelated to this project and may be years older, so it is only + # a last resort even when a wide pragma would accept it. + default_version = self._extract_version_from_solc_name(self.solc_default_version) + if default_version and shutil.which(self.solc_default_version): + candidates.append((self.solc_default_version, default_version)) + plain_version = self._get_plain_solc_version() + if plain_version: + candidates.append(("solc", plain_version)) + self._solc_candidates = candidates + return self._solc_candidates + + def _plan_solc_fallback( + self, output: str, updated_config_dict: Dict, contracts: List[ContractHandle] + ) -> Optional[SolcFallbackPlan]: + """The plan for the missing binary named in ``output``: for each contract + pinned to it in ``compiler_map``, the first installed compiler its pragma + admits, or an entry in ``blocked`` when no candidate qualifies. + + None when ``output`` reports no missing binary. An unreadable or unparseable + pragma is no evidence of a conflict, so those contracts take the first + candidate. + """ + failed_solc = self._detect_solc_not_found(output) + if failed_solc is None: + return None - if shutil.which(self.solc_default_version): - return self.solc_default_version + cmap = updated_config_dict.get("compiler_map") or {} + pinned = [name for name, version in cmap.items() if version == failed_solc] + pragma_by_contract = { + handle.contract_name: read_pragma_from_source_file(Path(handle.source_file), self.project_root) + for handle in contracts + } + candidates = self._solc_fallback_candidates() + + compliant: Dict[str, str] = {} + blocked: List[BlockedSolcPin] = [] + for name in pinned: + pragma = pragma_by_contract.get(name) + for binary, version in candidates: + if pragma is None or pragma_admits(pragma, version) is not False: + compliant[name] = binary + break + else: + blocked.append(BlockedSolcPin(contract_name=name, pragma_spec=pragma or "unknown")) - return "solc" + return SolcFallbackPlan(failed_solc=failed_solc, compliant=compliant, blocked=tuple(blocked)) @staticmethod def _extract_version_from_solc_name(solc_name: str) -> Optional[str]: @@ -1577,11 +1950,37 @@ def apply_compiler_version_workaround( return True def _apply_via_ir_workaround(self, contract_needing_via_ir: str, config_dict: Dict) -> Dict: - """Add solc_via_ir_map entry for contract that needs via-ir compilation.""" - # solc_via_ir_map is seeded up front by _seed_compile_maps; just set the - # contract that needs via-ir to True. + """Enable via-ir for the contract that needs it — or, past the point where naming + them one at a time pays off, for the whole scene. + + Per contract is the better answer while the count is small: every contract left on + legacy codegen keeps the internal-function summaries via-ir might inline away. But + each one costs a full compile of the scene to discover, so a project that needs it + widely spends dozens of compiles walking there. Past VIA_IR_SCENE_THRESHOLD the + exposure is already broad and the rest of the scene is likely to follow, so the + remaining contracts are switched in one step. A project whose build config declares + via-ir skips the walk entirely — it has told us where this ends. + """ + # solc_via_ir_map is seeded up front by _seed_compile_maps. + self._via_ir_contracts.add(contract_needing_via_ir) config_dict["solc_via_ir_map"][contract_needing_via_ir] = True - self.log(f"Adding via-ir workaround for contract: {contract_needing_via_ir}") + + scene_wide = self.declared_via_ir or len(self._via_ir_contracts) >= VIA_IR_SCENE_THRESHOLD + if scene_wide and not all(config_dict["solc_via_ir_map"].values()): + for name in config_dict["solc_via_ir_map"]: + config_dict["solc_via_ir_map"][name] = True + reason = ( + "the build config declares via-ir" + if self.declared_via_ir + else f"{len(self._via_ir_contracts)} contracts have needed it individually" + ) + self.log( + f"Enabling via-ir for the whole scene ({reason}); the remaining contracts " + f"lose their internal-function summaries too", + "WARNING", + ) + else: + self.log(f"Adding via-ir workaround for contract: {contract_needing_via_ir}") return config_dict @@ -1626,5 +2025,8 @@ def _build_packages_from_remapping_sources(self) -> List[str]: """ # Read from the directory that owns the build config (the run root unless the # contract lives in a monorepo sub-project); the helper resolves every relative - # target absolute against it, so the emitted paths are valid from the run CWD. - return build_packages_from_remapping_sources(base_dir=self.build_config_dir, log_fn=self.log) + # target absolute against it, so the emitted paths are valid from the run CWD, and + # re-expresses remapping contexts against the run root, where solc matches them. + return build_packages_from_remapping_sources( + base_dir=self.build_config_dir, log_fn=self.log, run_root=self.project_root + ) diff --git a/certora_autosetup/utils/constants.py b/certora_autosetup/utils/constants.py index 957ff1a9..3c16e63b 100644 --- a/certora_autosetup/utils/constants.py +++ b/certora_autosetup/utils/constants.py @@ -70,6 +70,7 @@ class LLMBackend(str, Enum): FILE_AUTOSETUP_RESULT = "autosetup_result.json" FILE_LLM_USAGE = "llm_usage.json" FILE_PROVER_USAGE = "prover_usage.json" +FILE_SUMMARIZATION_CANDIDATES = "summarization_candidates.json" # Compiled-scene method inventory emitted under .certora_internal/ FILE_ALL_METHODS_JSON = "all_methods.json" diff --git a/certora_autosetup/utils/contract_utils.py b/certora_autosetup/utils/contract_utils.py index 50398773..bf3e9939 100644 --- a/certora_autosetup/utils/contract_utils.py +++ b/certora_autosetup/utils/contract_utils.py @@ -145,6 +145,20 @@ def deduplicate_contract_handles(handles: list[ContractHandle]) -> list[Contract return result +def with_contract_handle( + handles: list[ContractHandle], to_add: ContractHandle +) -> list[ContractHandle]: + """Return *handles* containing *to_add*, appended if it is not already there. + + Any handle carrying the same ``contract_name`` as *to_add* but a different source file + is dropped, so the returned list names each contract once: two handles sharing a name + make the scene ambiguous about which file it came from. + """ + if to_add in handles: + return handles + return [h for h in handles if h.contract_name != to_add.contract_name] + [to_add] + + def resolve_contract_handles( contract_handles: list[ContractHandle], project_root: Path, diff --git a/certora_autosetup/utils/file_utils.py b/certora_autosetup/utils/file_utils.py index 39b9b423..a3f3ed3b 100644 --- a/certora_autosetup/utils/file_utils.py +++ b/certora_autosetup/utils/file_utils.py @@ -4,23 +4,9 @@ import os import threading import uuid -from collections.abc import Iterator from pathlib import Path from typing import Any -import ijson - - -def stream_ast_files(ast_path: Path) -> Iterator[tuple[str, Any]]: - """Yield ``(relative_path, path_data)`` pairs from a ``.asts.json``. - - The file is streamed one top-level entry at a time, so only a single source - file's ASTs are held in memory. ``.asts.json`` is sometimes many GB. - Structure: ``dict[relative_path: dict[absolute_path: dict[node_id: node_data]]]``. - """ - with open(ast_path, "rb") as f: - yield from ijson.kvitems(f, "") - def atomic_write_json(file_path: Path, data: Any, indent: int = 2) -> None: """ diff --git a/certora_autosetup/utils/import_diagnostics.py b/certora_autosetup/utils/import_diagnostics.py new file mode 100644 index 00000000..86ad1766 --- /dev/null +++ b/certora_autosetup/utils/import_diagnostics.py @@ -0,0 +1,304 @@ +"""Classify solc's ``ParserError: Source "…" not found`` failures against the conf's packages. + +A source-not-found failure has several distinct causes with different remedies, and the +distinction is decidable from the filesystem plus the packages list the conf actually carried: +solc reports the source unit name *after* remapping, so a prefix match of the reported name +against a package's target says whether a remapping fired, and ``is_dir()`` on that target — and, +when it is absent, on the ``node_modules/`` above it — says whether the package is installed +at all. Anything the evidence does not decide stays ``UNMAPPED_IMPORT`` rather than being guessed +at. + +What this deliberately does NOT try to say: why a package is missing (autosetup does not run the +dependency install and never sees its exit status), which version was intended when several +ancestors provide a package, or whether a file missing inside an installed package means a wrong +remapping suffix or a version mismatch — the evidence is identical for both. solc also truncates +its error list, so the absence of a class from one output is not evidence that it does not occur. +""" + +import os +import re +from dataclasses import dataclass +from enum import StrEnum +from pathlib import Path +from typing import Dict, List, Optional, Tuple + +from certora_autosetup.utils.remappings import node_modules_package_root + +# A solc source-location line, e.g. `` --> contracts/Foo.sol:120:9:``. It names the offending +# file in a whole-project (non-autofinder) solc error, where there is no ``Compiling ...`` +# progress line to recover it from. +_SOURCE_LOCATION_RE = re.compile(r"^\s*-->\s+(?P.+?):\d+:\d+:?\s*$") + + +def path_from_source_location_line(line: str) -> Optional[str]: + """Return ```` from a solc `` --> :::`` source-location line, or + None if ``line`` isn't one.""" + match = _SOURCE_LOCATION_RE.match(line) + return match.group("path") if match else None + + +# Wrap-tolerant: solc hard-wraps its diagnostics at a fixed width, so the phrase is regularly +# split across newlines. Matched against whitespace-normalized output (see `_normalize_with_lines`). +_SOURCE_NOT_FOUND_RE = re.compile(r'ParserError:\s+Source\s+"(?P[^"]+)"\s+not\s+found') + +# certoraRun prefixes the diagnostic with the importing file: `path/Foo.sol:12:5: ParserError: …`. +_IMPORTER_PREFIX_RE = re.compile(r"(?P\S+):\d+:\d+:\s*$") + +# How many lines after the error to look for solc's `--> ::` importer line. +_IMPORTER_LOOKAHEAD = 6 + + +class UnresolvedImportKind(StrEnum): + """Why one ``Source "…" not found`` happened, as far as the filesystem can decide.""" + + # A remapping fired and neither its target directory nor the `node_modules/` it names + # exists: the dependency is not installed where the packages list points. This is the class + # the ancestor walk resolves. + PACKAGE_TARGET_MISSING = "package_target_missing" + # A remapping fired, the `node_modules/` it names is installed, but the remapped + # subdirectory inside it is absent — the same state `resolve_node_modules_target` reports as + # `subpath_missing`, reached only when no ancestor had the whole target either. + PACKAGE_SUBPATH_MISSING = "package_subpath_missing" + # A remapping fired, its target directory exists, but the file inside it does not. Rebuilding + # the packages list cannot help — the suffix or the installed version is wrong. + FILE_MISSING_IN_PACKAGE = "file_missing_in_package" + # No package entry covers the source unit name, and it names no directory in the project tree. + UNMAPPED_IMPORT = "unmapped_import" + # The source unit is inside the project's own tree: a missing or misspelled project file. + MISSING_PROJECT_FILE = "missing_project_file" + + +@dataclass +class UnresolvedImport: + """One unresolved import, with the package entry (if any) that governs it.""" + + source_unit: str + kind: UnresolvedImportKind + package_key: Optional[str] = None + package_target: Optional[str] = None + # The installed `node_modules/` above a target that does not exist; set on + # PACKAGE_SUBPATH_MISSING only, where it is what separates that class from a missing package. + package_root: Optional[str] = None + importer: Optional[str] = None + hint: Optional[str] = None + + +def _normalize_with_lines(output: str) -> Tuple[str, List[int]]: + """Whitespace-normalized output plus, per character, the line it came from. + + The line map is what lets a match found in the normalized text be located back in the raw + output, where the ``-->`` importer line still exists as its own line. + """ + parts: List[str] = [] + line_of_char: List[int] = [] + for index, line in enumerate(output.splitlines()): + for token in line.split(): + if parts: + parts.append(" ") + line_of_char.append(index) + parts.append(token) + line_of_char.extend([index] * len(token)) + return "".join(parts), line_of_char + + +def parse_unresolved_imports(output: str) -> List[Tuple[str, Optional[str]]]: + """Return ``(source_unit, importer)`` for every source-not-found error in ``output``. + + ``importer`` is the file whose import failed, taken from solc's ``--> ::`` + location line when there is one, or from the ``:::`` prefix certoraRun puts + in front of the diagnostic. It is None when neither is printed. + """ + normalized, line_of_char = _normalize_with_lines(output) + raw_lines = output.splitlines() + + matches = list(_SOURCE_NOT_FOUND_RE.finditer(normalized)) + results: List[Tuple[str, Optional[str]]] = [] + for position, match in enumerate(matches): + start_line = line_of_char[match.start()] if match.start() < len(line_of_char) else 0 + next_line = ( + line_of_char[matches[position + 1].start()] + if position + 1 < len(matches) + else len(raw_lines) + ) + importer: Optional[str] = None + for line in raw_lines[start_line + 1: min(start_line + 1 + _IMPORTER_LOOKAHEAD, next_line)]: + importer = path_from_source_location_line(line) + if importer is not None: + break + if importer is None: + prefix_match = _IMPORTER_PREFIX_RE.search(normalized[:match.start()]) + if prefix_match: + importer = prefix_match.group("path") + results.append((match.group("src"), importer)) + return results + + +def _absolute(path: str, run_root: Path) -> str: + """Absolute, textually normalized form of a possibly run-root-relative path.""" + return os.path.normpath(path if os.path.isabs(path) else os.path.join(str(run_root), path)) + + +def _split_package(entry: str) -> Optional[Tuple[str, str, str]]: + """Split a packages entry into ``(context, prefix, target)``; None if it has no ``=``.""" + if "=" not in entry: + return None + key, target = entry.split("=", 1) + context, _, prefix = key.rpartition(":") + return context, prefix, target + + +def _is_under(child: str, parent: str) -> bool: + """True when ``child`` is ``parent`` itself or lives inside it (textual, both absolute).""" + parent = parent.rstrip("/") + return child == parent or child.startswith(parent + "/") + + +def classify_unresolved_import( + source_unit: str, + packages: List[str], + run_root: Path, + importer: Optional[str] = None, +) -> UnresolvedImport: + """Decide why ``source_unit`` did not resolve, given the packages the conf carried. + + solc names the source unit *after* applying a remapping, so a source unit living under a + package's target proves that package's remapping fired — and then a single ``is_dir()`` on + the target separates "the dependency is not installed there" from "the dependency is there + but this file is not". When no target covers the source unit, a key that textually prefixes + it means a remapping was declared but did not apply, which for a context-scoped key is + explained by the context not prefixing the importer. + """ + absolute_source = _absolute(source_unit, run_root) + + # Longest matching target wins, so nested package targets classify against the one that + # actually produced this source unit name. + best: Optional[Tuple[str, str, str]] = None + for entry in packages: + split = _split_package(entry) + if split is None: + continue + _, prefix, target = split + absolute_target = _absolute(target, run_root) + if _is_under(absolute_source, absolute_target) and ( + best is None or len(absolute_target) > len(_absolute(best[2], run_root)) + ): + best = (entry.split("=", 1)[0], prefix, target) + + if best is not None: + key, _, target = best + absolute_target = _absolute(target, run_root) + # A target that does not exist has two very different causes, and the package root above + # it decides which: no `node_modules/` at all (nothing is installed) versus an + # installed package whose remapped subdirectory is absent. Without this split the second + # case is described as "the dependency is not installed", contradicting + # `resolve_node_modules_target`, which already found the package directory. + package_root = node_modules_package_root(absolute_target) + if Path(absolute_target).is_dir(): + kind = UnresolvedImportKind.FILE_MISSING_IN_PACKAGE + package_root = None + elif ( + package_root is not None + and package_root != absolute_target + and Path(package_root).is_dir() + ): + kind = UnresolvedImportKind.PACKAGE_SUBPATH_MISSING + else: + kind = UnresolvedImportKind.PACKAGE_TARGET_MISSING + package_root = None + return UnresolvedImport( + source_unit=source_unit, + kind=kind, + package_key=key, + package_target=target, + package_root=package_root, + importer=importer, + ) + + # No target covers it, but a declared key does: the remapping exists and did not apply. + for entry in packages: + split = _split_package(entry) + if split is None: + continue + context, prefix, _ = split + if not prefix or not source_unit.startswith(prefix): + continue + hint = None + if context and importer is not None and not importer.startswith(context): + hint = ( + f"remapping '{context}:{prefix}' is scoped to context '{context}', which does not " + f"prefix the importing file '{importer}', so it never applied" + ) + elif context: + hint = ( + f"remapping '{context}:{prefix}' is scoped to context '{context}'; solc did not " + f"name the importing file, so whether the context applied is undecidable here" + ) + return UnresolvedImport( + source_unit=source_unit, + kind=UnresolvedImportKind.UNMAPPED_IMPORT, + package_key=entry.split("=", 1)[0], + importer=importer, + hint=hint, + ) + + first_segment = source_unit.replace(os.sep, "/").split("/")[0] + if first_segment and (run_root / first_segment).is_dir(): + return UnresolvedImport( + source_unit=source_unit, + kind=UnresolvedImportKind.MISSING_PROJECT_FILE, + importer=importer, + ) + + return UnresolvedImport( + source_unit=source_unit, + kind=UnresolvedImportKind.UNMAPPED_IMPORT, + importer=importer, + ) + + +def _describe_one(failure: UnresolvedImport) -> str: + """One line naming the source unit and the remedy its class implies.""" + if failure.kind == UnresolvedImportKind.PACKAGE_TARGET_MISSING: + return ( + f'"{failure.source_unit}": package \'{failure.package_key}\' maps to ' + f"{failure.package_target}, which does not exist — the dependency is not installed " + f"there and was not found in any ancestor node_modules up to the run root" + ) + if failure.kind == UnresolvedImportKind.PACKAGE_SUBPATH_MISSING: + return ( + f'"{failure.source_unit}": package \'{failure.package_key}\' maps to ' + f"{failure.package_target}; the package is installed at {failure.package_root} but " + f"the remapped subdirectory inside it is missing — the remapping suffix or the " + f"installed version is wrong" + ) + if failure.kind == UnresolvedImportKind.FILE_MISSING_IN_PACKAGE: + return ( + f'"{failure.source_unit}": package \'{failure.package_key}\' maps to ' + f"{failure.package_target}, which exists but does not contain this file — the " + f"remapping suffix or the installed version is wrong, so rebuilding the packages " + f"list cannot help" + ) + if failure.kind == UnresolvedImportKind.MISSING_PROJECT_FILE: + return ( + f'"{failure.source_unit}": inside the project tree but absent — a missing or ' + f"misspelled project file, not a dependency problem" + ) + detail = f" ({failure.hint})" if failure.hint else "" + return ( + f'"{failure.source_unit}": no package entry resolves this import{detail}' + ) + + +def describe_unresolved_imports(failures: List[UnresolvedImport]) -> str: + """Human-readable summary, grouped by kind so a long failure list stays readable.""" + if not failures: + return "" + by_kind: Dict[UnresolvedImportKind, List[UnresolvedImport]] = {} + for failure in failures: + by_kind.setdefault(failure.kind, []).append(failure) + + lines: List[str] = [] + for kind, group in by_kind.items(): + lines.append(f"{kind.value} ({len(group)}):") + lines.extend(f" - {_describe_one(failure)}" for failure in group) + return "\n".join(lines) diff --git a/certora_autosetup/utils/paths.py b/certora_autosetup/utils/paths.py index 930be449..1e69a825 100644 --- a/certora_autosetup/utils/paths.py +++ b/certora_autosetup/utils/paths.py @@ -27,6 +27,7 @@ FILE_ERC7201_SPEC, FILE_LLM_USAGE, FILE_PROVER_USAGE, + FILE_SUMMARIZATION_CANDIDATES, SUMMARIES_SUBDIR, ) @@ -144,3 +145,9 @@ def resolve_autosetup_prover_usage_file(project_root: Path) -> Path | None: """Locate the ``prover_usage.json`` the most recent autosetup run wrote under ``project_root`` (``None`` if absent). See :func:`_resolve_autosetup_reports_file`.""" return _resolve_autosetup_reports_file(project_root, FILE_PROVER_USAGE) + + +def resolve_autosetup_summarization_candidates_file(project_root: Path) -> Path | None: + """Locate the ``summarization_candidates.json`` the most recent autosetup run wrote under + ``project_root`` (``None`` if absent). See :func:`_resolve_autosetup_reports_file`.""" + return _resolve_autosetup_reports_file(project_root, FILE_SUMMARIZATION_CANDIDATES) diff --git a/certora_autosetup/utils/project_dir.py b/certora_autosetup/utils/project_dir.py index 118af805..61f81f29 100644 --- a/certora_autosetup/utils/project_dir.py +++ b/certora_autosetup/utils/project_dir.py @@ -27,37 +27,64 @@ "truffle.js", ) -# Truffle writes artifacts here unless the config sets `contracts_build_directory`. Reading -# that override means running node against the config, which is more than this needs: it -# only has to recognise a built project, and the manager resolves the real path later. +# Hardhat writes artifacts here unless the config sets `paths.artifacts`, and Truffle +# unless it sets `contracts_build_directory`. Reading either override means running node +# against the project's own config (see `HardhatManager._extract_config_via_node`), which +# is more than these helpers need: they only have to recognise a built project, and the +# managers resolve the real path later. Foundry's `out` is the one override read here, +# because a TOML parse is easy. +HARDHAT_DEFAULT_ARTIFACT_DIR = Path("artifacts") TRUFFLE_DEFAULT_BUILD_DIR = Path("build") / "contracts" -def _artifact_dir_of(config_dir: Path) -> Optional[Path]: - """Where *config_dir*'s build system would put artifacts, or None if it holds no config. +def foundry_artifact_dir(config_dir: Path) -> Optional[Path]: + """Where a Foundry project at *config_dir* writes artifacts, or None if it has no config. - Foundry's ``out`` is configurable, so read it when present; Hardhat's ``artifacts`` and - Truffle's ``build/contracts`` are taken as defaults. The directory is not required to + Honors the ``out`` setting of the default profile. The directory is not required to exist — the caller tests that. """ foundry_toml = config_dir / "foundry.toml" - if foundry_toml.exists(): - out = "out" - try: - with foundry_toml.open("rb") as f: - data = tomllib.load(f) - profiles = data.get("profile", {}) - out = profiles.get("default", {}).get("out") or data.get("out") or "out" - except (tomllib.TOMLDecodeError, OSError): - pass - return config_dir / out + if not foundry_toml.exists(): + return None + out = "out" + try: + with foundry_toml.open("rb") as f: + data = tomllib.load(f) + profiles = data.get("profile", {}) + out = profiles.get("default", {}).get("out") or data.get("out") or "out" + except (tomllib.TOMLDecodeError, OSError): + pass + return config_dir / out + + +def hardhat_artifact_dir(config_dir: Path) -> Optional[Path]: + """Where a Hardhat project at *config_dir* writes artifacts, or None if it has no config.""" if (config_dir / "hardhat.config.js").exists() or (config_dir / "hardhat.config.ts").exists(): - return config_dir / "artifacts" + return config_dir / HARDHAT_DEFAULT_ARTIFACT_DIR + return None + + +def truffle_artifact_dir(config_dir: Path) -> Optional[Path]: + """Where a Truffle project at *config_dir* writes artifacts, or None if it has no config.""" if (config_dir / "truffle-config.js").exists() or (config_dir / "truffle.js").exists(): return config_dir / TRUFFLE_DEFAULT_BUILD_DIR return None +def _artifact_dir_of(config_dir: Path) -> Optional[Path]: + """Where *config_dir*'s build system would put artifacts, or None if it holds no config. + + A directory holding several configs answers for the first of them, in the same order + ``BuildSystemDetector`` ranks them: any of the three is enough to recognise the + directory as a project that got built, which is all the caller asks. + """ + return ( + foundry_artifact_dir(config_dir) + or hardhat_artifact_dir(config_dir) + or truffle_artifact_dir(config_dir) + ) + + def find_build_config_dir(contract_path: Path, root: Path) -> Path: """Return the directory whose build system actually produced *contract_path*'s artifacts. diff --git a/certora_autosetup/utils/remappings.py b/certora_autosetup/utils/remappings.py index 47d76f42..ab2e0f1b 100644 --- a/certora_autosetup/utils/remappings.py +++ b/certora_autosetup/utils/remappings.py @@ -10,8 +10,9 @@ import json import os import subprocess +from dataclasses import dataclass, field from pathlib import Path -from typing import Callable, Dict, List +from typing import Callable, Dict, List, Literal, Optional, Tuple import tomllib @@ -22,8 +23,179 @@ # prefix, so they must not receive a trailing-slash boundary (see `_merge_remapping_entry`). _SOURCE_SUFFIXES = (".sol", ".vy", ".yul") +# The one directory name npm/yarn/pnpm hoist packages into, and the only target prefix the +# ancestor walk applies to (see `resolve_node_modules_target`). +_NODE_MODULES = "node_modules" -def build_packages_from_remapping_sources(base_dir: Path, log_fn: LogFn, profile: str = "default") -> List[str]: + +@dataclass +class PackageResolution: + """Where a remapping target actually lives, and how it was found. + + ``path`` is the target to emit (always an absolute path built from ``base_dir`` or one of + its ancestors). ``kind`` says which candidate answered: + - ``local`` — resolved under ``base_dir``, i.e. the target as authored; + - ``hoisted`` — resolved in an ancestor's ``node_modules`` (a hoisted install); + - ``subpath_missing`` — a ``node_modules/`` was found, but the remapped subdirectory + inside it is absent, so the target names a directory that does not exist; + - ``unresolved`` — no ancestor up to the run root has the package at all; ``path`` is the + base-dir target, unchanged. + ``package_dir`` is the ``node_modules/`` directory the resolution came from (None when + nothing was found), and ``searched`` lists every candidate tested, for the log message. + """ + + path: str + kind: Literal["local", "hoisted", "subpath_missing", "unresolved"] + package_dir: Optional[str] = None + searched: List[str] = field(default_factory=list) + + +def _split_node_modules_target(target: str) -> Optional[Tuple[str, str]]: + """Split a bare ``node_modules/[/]`` target into ``(pkg, subpath)``. + + Returns None for anything else, which is what keeps the ancestor walk narrow. A target + that spells a location before ``node_modules`` (``packages/a/node_modules/x``, + ``../vendor/node_modules/x``) names a specific install the author chose, so it is left + alone; only a bare ``node_modules/...`` is the node-resolution idiom that hoisting applies + to. ```` takes two segments for a scoped package (``@scope/name``), one otherwise. + """ + normalized = target.replace(os.sep, "/").replace("\\", "/") + segments = [s for s in normalized.split("/") if s and s != "."] + if not segments or segments[0] != _NODE_MODULES: + return None + rest = segments[1:] + if not rest: + return None + pkg_len = 2 if rest[0].startswith("@") else 1 + if len(rest) < pkg_len: + return None + return "/".join(rest[:pkg_len]), "/".join(rest[pkg_len:]) + + +def node_modules_package_root(target: str) -> Optional[str]: + """The ``…/node_modules/`` prefix of ``target``, or None when it names no package. + + Unlike ``_split_node_modules_target`` this accepts a target with anything in front of + ``node_modules`` (an absolute path, a sub-project prefix), because its callers are handed a + finished packages entry rather than an authored remapping. The *last* ``node_modules`` + segment wins: with a nested install (``a/node_modules/b/node_modules/c``) the innermost one + is what governs the tail. ```` takes two segments for a scoped package (``@scope/name``), + one otherwise; the result equals ``target`` when the target IS the package root, which is how + a caller tells "no subdirectory was remapped" apart from one that was. + """ + segments = [s for s in target.replace(os.sep, "/").replace("\\", "/").split("/") if s != "."] + if _NODE_MODULES not in segments: + return None + index = len(segments) - 1 - segments[::-1].index(_NODE_MODULES) + rest = segments[index + 1:] + if not rest: + return None + pkg_len = 2 if rest[0].startswith("@") else 1 + if len(rest) < pkg_len: + return None + return "/".join(segments[: index + 1 + pkg_len]) + + +def _ancestor_roots(base_dir: Path, run_root: Optional[Path]) -> List[str]: + """Directories to search for a hoisted package: ``base_dir`` first, then each parent up to + and including ``run_root``. + + The chain has a single element — today's behaviour exactly — when there is no run root or + when ``base_dir`` is not inside it. + + Both the containment test and the walk are textual (``os.path.normpath`` / + ``os.path.dirname``, never ``Path.resolve``), and they must stay that way *together*: + ``BuildSystemConfig._relativize_packages`` makes every emitted package path relative with a + textual ``Path.relative_to(project_root)``, so a resolved path (``/tmp`` → ``/private/tmp`` + on macOS) would stop matching the run root and fall back to absolute paths. ``_rebase_context`` + makes the same textual assumption with ``os.path.relpath``. Mixing the two — a step count + taken from resolved paths, candidates composed textually — makes the walk escape the run root + when ``base_dir`` reaches it through a symlink (a candidate above the run root names a + directory certoraRun does not upload) and stop short of it in the opposite case (a hoisted + package silently missed). Each candidate is therefore derived from the previous one and the + loop ends on the run root itself. + """ + if run_root is None: + return [str(base_dir)] + root = os.path.normpath(str(run_root)) + candidate = os.path.normpath(str(base_dir)) + if candidate != root and not candidate.startswith(root.rstrip(os.sep) + os.sep): + return [str(base_dir)] + roots = [candidate] + while candidate != root: + parent = os.path.dirname(candidate) + if parent == candidate: + break + candidate = parent + roots.append(candidate) + return roots + + +def resolve_node_modules_target( + target: str, base_dir: Path, run_root: Optional[Path] +) -> PackageResolution: + """Resolve a relative remapping target, walking ancestor ``node_modules`` when needed. + + npm/yarn hoist a dependency to the highest ``node_modules`` that can satisfy every consumer, + so a sub-project's ``node_modules/`` frequently does not exist while the repo root's + does. solc has no such resolver: the packages list must name the directory. This reproduces + node's own order — nearest ``node_modules`` first, then outwards — bounded at the run root so + the emitted path stays inside the tree certoraRun uploads. + + Only bare ``node_modules/...`` targets are walked. forge (``lib/``) and soldeer + (``dependencies/``) do not hoist, and a sibling project's ``lib/`` is routinely a + different pin, so walking those would silently bind the wrong version. + + A target that resolves under ``base_dir`` is always returned unchanged, so the walk can only + change an entry whose current target does not exist on disk. When the nearest package lacks + the remapped subpath, a farther ancestor that has the whole target is preferred — a + deliberate deviation from node, since only the full target is usable by solc. + """ + split = _split_node_modules_target(target) + if split is None: + return PackageResolution(path=str(base_dir / target), kind="local") + + package, subpath = split + searched: List[str] = [] + nearest_package_dir: Optional[str] = None + + for index, root in enumerate(_ancestor_roots(base_dir, run_root)): + package_dir = os.path.join(root, _NODE_MODULES, package) + full_target = os.path.join(package_dir, *subpath.split("/")) if subpath else package_dir + searched.append(full_target) + if not Path(package_dir).is_dir(): + continue + if nearest_package_dir is None: + nearest_package_dir = package_dir + # `.exists()` rather than `.is_dir()`: a remapping may target a single source file. + if Path(full_target).exists(): + return PackageResolution( + path=full_target, + kind="local" if index == 0 else "hoisted", + package_dir=package_dir, + searched=searched, + ) + + if nearest_package_dir is not None: + full_target = ( + os.path.join(nearest_package_dir, *subpath.split("/")) if subpath else nearest_package_dir + ) + return PackageResolution( + path=full_target, + kind="subpath_missing", + package_dir=nearest_package_dir, + searched=searched, + ) + + return PackageResolution(path=str(base_dir / target), kind="unresolved", searched=searched) + + +def build_packages_from_remapping_sources( + base_dir: Path, + log_fn: LogFn, + profile: str = "default", + run_root: Optional[Path] = None, +) -> List[str]: """Build a merged packages list from forge remappings, foundry.toml, remappings.txt, package.json. All sources are read relative to ``base_dir`` (the Foundry project dir), and ``forge remappings`` @@ -32,6 +204,15 @@ def build_packages_from_remapping_sources(base_dir: Path, log_fn: LogFn, profile ``FOUNDRY_PROFILE`` and selects the ``[profile.]`` remappings read from foundry.toml when forge is unavailable, so a non-default profile's remappings are honored. + ``run_root`` is the directory certoraRun is invoked from (the autosetup run root). It bounds + both halves of a remapping. The *context* half must be expressed against it, because solc + matches contexts against source unit names — see ``_rebase_context``. The *target* half of a + bare ``node_modules/...`` entry is resolved against it too: the package is looked for in + ``base_dir/node_modules`` first and then in each ancestor's up to the run root, which is how + a hoisted install is found (see ``resolve_node_modules_target``). A target that resolves + under ``base_dir`` resolves to exactly the same path either way, so with ``run_root`` equal + to ``base_dir`` or absent the result is unchanged; pass it whenever the caller knows it. + Priority on key conflict (highest wins, with a warning on path mismatch): 1. ``forge remappings`` — recursively walks nested foundry.toml files (e.g. lib/*/foundry.toml) and emits paths relative to CWD; strictly stronger than parsing the top-level @@ -74,6 +255,7 @@ def build_packages_from_remapping_sources(base_dir: Path, log_fn: LogFn, profile remapping_key_to_source=remapping_key_to_source, warn_on_mismatch=False, base_dir=base_dir, + run_root=run_root, log_fn=log_fn, ) elif result is not None: @@ -111,6 +293,7 @@ def build_packages_from_remapping_sources(base_dir: Path, log_fn: LogFn, profile remapping_key_to_source=remapping_key_to_source, warn_on_mismatch=False, base_dir=base_dir, + run_root=run_root, log_fn=log_fn, ) @@ -128,6 +311,7 @@ def build_packages_from_remapping_sources(base_dir: Path, log_fn: LogFn, profile remapping_key_to_source=remapping_key_to_source, warn_on_mismatch=True, base_dir=base_dir, + run_root=run_root, log_fn=log_fn, ) @@ -148,12 +332,58 @@ def build_packages_from_remapping_sources(base_dir: Path, log_fn: LogFn, profile remapping_key_to_source=remapping_key_to_source, warn_on_mismatch=True, base_dir=base_dir, + run_root=run_root, log_fn=log_fn, ) return [f"{key}={path}" for key, path in remapping_key_to_path.items()] +def _rebase_context(context: str, base_dir: Path, run_root: Optional[Path], log_fn: LogFn) -> str: + """Re-express a remapping *context* against ``run_root``. + + A remapping's two halves are matched against different things. The target is a filesystem + path, so any spelling that reaches the directory works. The context is matched by solc + against the **source unit name** of the importing file — i.e. the path as it appears in the + conf's ``files``, which is relative to the directory certoraRun runs from. ``forge + remappings`` reports contexts relative to the Foundry project dir instead, so for a project + nested under the run root (``/chains/ethereum-1/foundry.toml``) a context like + ``src/Widget_1234/`` never prefixes the source unit name ``chains/ethereum-1/src/Widget_1234/…``, + no remapping applies, and every import of that sub-project fails to resolve. + + A context is rebased only when ``base_dir/context`` is a real directory, which is what tells + a project-relative context apart from one that is already run-root-relative. Anything else + (no ``run_root``, a context naming no directory, a context resolving to the run root itself + or outside it) is returned untouched: those are shapes this cannot improve on, so leave them + alone. Only the outside-the-run-root case warns — it is the one that cannot work at all. + """ + if run_root is None or not context: + return context + + resolved = Path(context) if Path(context).is_absolute() else base_dir / context + if not resolved.is_dir(): + return context + + rebased = os.path.relpath(resolved, run_root) + if rebased == os.curdir: + # The context IS the run root, so it already covers every source unit name. Its faithful + # translation is a global, context-free remapping — but promoting a scoped remapping to + # global would let it shadow correct global mappings (solc ranks longest matching context + # first), so keep it as authored. Nothing is wrong here, hence no warning. + return context + if rebased.startswith(os.pardir): + log_fn( + f"Remapping context '{context}' resolves outside the run root {run_root} " + f"— leaving it as is", + "WARNING", + ) + return context + + # relpath drops the trailing slash; the context is a path *prefix*, so put it back exactly + # when the source had one (see `_merge_remapping_entry` on why the boundary slash matters). + return rebased + "/" if context.endswith("/") else rebased + + def _merge_remapping_entry( *, entry: str, @@ -162,6 +392,7 @@ def _merge_remapping_entry( remapping_key_to_source: Dict[str, str], warn_on_mismatch: bool, base_dir: Path, + run_root: Optional[Path] = None, log_fn: LogFn, ) -> None: """Record a single `key=path` remapping entry into the running key->path/source maps. @@ -180,7 +411,9 @@ def _merge_remapping_entry( ``@oz/contracts/`` from different sources onto one key, so the dedup below stays correct.) Relative target paths are resolved to absolute against ``base_dir`` so the packages list is - valid even when the process CWD differs from the project dir. + valid even when the process CWD differs from the project dir. The *context* half of a + context-scoped key gets the opposite treatment — ``_rebase_context`` re-expresses it against + ``run_root``, because solc matches it against source unit names rather than the filesystem. On a key conflict (already populated by an earlier-priority source): - if ``warn_on_mismatch`` and the stored path differs from the new one, log a warning naming @@ -193,9 +426,38 @@ def _merge_remapping_entry( key = raw_key.strip() path = raw_path.strip() - # Resolve a relative target against base_dir first (Path() drops any trailing slash). + # A context-scoped key is `context:prefix`; solc reads the context as a source-unit-name + # prefix, so it belongs to the run root while the prefix is opaque text (see `_rebase_context`). + if ":" in key: + context, prefix = key.split(":", 1) + key = f"{_rebase_context(context, base_dir, run_root, log_fn)}:{prefix}" + + # Resolve a relative target against base_dir first (Path() drops any trailing slash), letting + # the ancestor walk find a hoisted node_modules package (see `resolve_node_modules_target`). if not Path(path).is_absolute(): - path = str(base_dir / path) + resolution = resolve_node_modules_target(path, base_dir, run_root) + if resolution.kind == "hoisted": + log_fn( + f"Package '{key}' is not installed under {base_dir}; resolving '{path}' to the " + f"hoisted install at {resolution.path}", + "INFO", + ) + elif resolution.kind == "subpath_missing": + log_fn( + f"Package '{key}': {resolution.package_dir} exists but the remapped subdirectory " + f"is missing (looked for {resolution.path}) — the remapping target may be wrong " + f"or the package is not built", + "WARNING", + ) + elif resolution.kind == "unresolved": + log_fn( + f"Package '{key}' target {resolution.path} does not exist " + f"(searched: {', '.join(resolution.searched)}) — the dependency is not installed " + f"under the project or any ancestor up to the run root; keeping the entry so solc " + f"reports the exact missing source", + "WARNING", + ) + path = resolution.path # Canonicalize a DIRECTORY remapping to a trailing-slash form so the key's prefix boundary is # preserved (see docstring) and key/path agree on that boundary. A remapping that targets a diff --git a/certora_autosetup/utils/solc_version_resolver.py b/certora_autosetup/utils/solc_version_resolver.py index 107b19f6..cf9abe27 100644 --- a/certora_autosetup/utils/solc_version_resolver.py +++ b/certora_autosetup/utils/solc_version_resolver.py @@ -113,11 +113,15 @@ def parse_pragma_constraint(pragma_spec: str) -> Optional[SpecifierSet]: Convert pragma solidity specification to packaging.SpecifierSet. Handles: - - Exact version: "0.8.26" -> "==0.8.26" + - Exact version: "0.8.26" or "=0.8.26" -> "==0.8.26" - Caret: "^0.8.0" -> ">=0.8.0,<0.9.0" + - Tilde: "~0.8.0" -> ">=0.8.0,<0.9.0" - Range: ">=0.8.0 <0.8.6" -> ">=0.8.0,<0.8.6" - GTE/LTE: ">=0.8.0" -> ">=0.8.0" + Disjunctions ("^0.6.0 || ^0.8.0") return None: a SpecifierSet is a conjunction + and cannot express them. + Args: pragma_spec: Raw pragma specification string @@ -125,9 +129,20 @@ def parse_pragma_constraint(pragma_spec: str) -> Optional[SpecifierSet]: SpecifierSet object or None if parsing fails """ try: - # Handle exact version: "0.8.26" - if re.match(r"^\d+\.\d+\.\d+$", pragma_spec): - return SpecifierSet(f"=={pragma_spec}") + if "||" in pragma_spec: + return None + + # Handle exact version, with or without the explicit "=": "0.8.26", "=0.8.26" + if re.match(r"^=?\d+\.\d+\.\d+$", pragma_spec): + return SpecifierSet(f"=={pragma_spec.lstrip('=')}") + + # Handle tilde: "~0.8.0" -> ">=0.8.0,<0.9.0". solc follows npm semantics, + # where a 0.x tilde floats the patch only. + tilde_match = re.match(r"^~(\d+)\.(\d+)\.(\d+)$", pragma_spec) + if tilde_match: + major, minor, patch = tilde_match.groups() + next_minor = int(minor) + 1 + return SpecifierSet(f">={major}.{minor}.{patch},<{major}.{next_minor}.0") # Handle caret: "^0.8.0" -> ">=0.8.0,<0.9.0" caret_match = re.match(r"^\^(\d+)\.(\d+)\.(\d+)$", pragma_spec) @@ -155,6 +170,26 @@ def parse_pragma_constraint(pragma_spec: str) -> Optional[SpecifierSet]: return None +def pragma_admits(pragma_spec: str, version: str) -> Optional[bool]: + """Whether ``version`` satisfies ``pragma_spec``. + + ``None`` means the spec could not be parsed into a constraint — unknown, which + callers must treat as "no evidence either way" rather than as a contradiction. + + Examples: + pragma_admits("0.6.4", "0.8.34") -> False (exact pin, cannot be widened) + pragma_admits("^0.8.0", "0.8.34") -> True + pragma_admits("^0.6.0 || ^0.8.0", "0.8.34") -> None (disjunction: not one constraint) + """ + constraint = parse_pragma_constraint(pragma_spec) + if constraint is None: + return None + try: + return Version(version) in constraint + except Exception: + return None + + def extract_pragma_spec(text: str) -> str | None: """ Extract pragma solidity specification from source code or error output. @@ -195,7 +230,9 @@ def read_pragma_from_source_file(source_file: Path, project_root: Optional[Path] source_file = project_root / source_file try: return extract_pragma_spec(source_file.read_text()) - except OSError: + except (OSError, UnicodeDecodeError): + # A source carrying non-UTF-8 bytes (an accented name in a header comment is + # the usual cause) reads as "pragma unknown" rather than failing the caller. return None diff --git a/composer/audit/store.py b/composer/audit/store.py index b758b6cf..26deeecc 100644 --- a/composer/audit/store.py +++ b/composer/audit/store.py @@ -68,7 +68,7 @@ class StoredSystemBinary(TypedDict): class StoredRunInfo(TypedDict): - spec: StoredSpecFile + specs: list[StoredSpecFile] interface_name: str interface_contents: str system_name: str @@ -204,8 +204,9 @@ def bytes_contents(self) -> bytes: class ResumeArtifact: """Bundle of everything needed to resume a prior run: the final interface - and system views, the spec file that was in play at completion, the full - final VFS, and the commentary the LLM attached on completion. + and system views, every spec file that was in play at completion (keyed by + VFS path), the full final VFS, and the commentary the LLM attached on + completion. All file-shaped fields satisfy ``Uploadable``; the executor passes them through its ``FileUploader`` on resume to rehydrate into ``Document`` / @@ -214,19 +215,29 @@ class ResumeArtifact: def __init__( self, final_intf: _StoredText, - spec_entry: ResumeSpecEntry, + spec_entries: list[ResumeSpecEntry], system_doc: "_StoredText | _StoredBinary", commentary: str, intf_path: str, vfs_cur: VFSRetriever, ): self.intf_vfs_handle = final_intf - self.spec = spec_entry + self._specs: dict[str, ResumeSpecEntry] = {s.vfs_path: s for s in spec_entries} self.system_vfs_handle: "_StoredText | _StoredBinary" = system_doc self.vfs = vfs_cur self.commentary = commentary self.interface_path = intf_path + @property + def spec_vfs_paths(self) -> list[str]: + """VFS paths of every spec registered with the run.""" + return list(self._specs.keys()) + + def spec_at(self, vfs_path: str) -> ResumeSpecEntry | None: + """The spec registered at ``vfs_path`` (final contents at completion), + or ``None`` if no spec was registered there.""" + return self._specs.get(vfs_path) + @cached_property def interface_file(self) -> str: return self.intf_vfs_handle.string_contents @@ -313,16 +324,15 @@ def _run_meta_ns(self) -> tuple[str, ...]: async def register_run( self, thread_id: str, - spec_vfs_path: str, - spec_file: TextDocument, + specs: list[tuple[str, TextDocument]], interface_file: TextDocument, system_doc: Document, vfs_init: Iterable[tuple[str, bytes]], reqs: list[str] | None, description: str | None = None, ) -> None: - """``spec_vfs_path`` is where the spec lives in the VFS (codegen's - historical convention is ``rules.spec``). + """``specs`` pairs each spec's VFS path with its document (codegen's + single-spec convention is one entry at ``rules.spec``). Spec/interface contents persist as plain strings (text-guaranteed upstream). The system doc is classified once here: ``string_contents`` @@ -330,11 +340,14 @@ async def register_run( binary record. ``description`` is free-form user-supplied text recorded on the ``run_meta`` slot so callers can find a run by name after the thread id has been lost.""" - stored_spec: StoredSpecFile = { - "vfs_path": spec_vfs_path, - "basename": spec_file.basename, - "contents": spec_file.string_contents, - } + stored_specs: list[StoredSpecFile] = [ + { + "vfs_path": vfs_path, + "basename": doc.basename, + "contents": doc.string_contents, + } + for (vfs_path, doc) in specs + ] system_text = system_doc.string_contents stored_system: str | StoredSystemBinary if system_text is not None: @@ -345,7 +358,7 @@ async def register_run( "contents": base64.standard_b64encode(system_doc.bytes_contents).decode("utf-8"), } run_info: StoredRunInfo = { - "spec": stored_spec, + "specs": stored_specs, "interface_name": interface_file.basename, "interface_contents": interface_file.string_contents, "system_name": system_doc.basename, @@ -402,25 +415,26 @@ async def get_resume_artifact(self, thread_id: str) -> ResumeArtifact: f"Resume artifact references {ra['interface_path']} but it's not in vfs_result" ) - # Pull the registered spec's *final* contents from the completed VFS. - # Missing is a hard error — it was present at registration time. - stored_spec = ri["spec"] - spec_vfs_path = stored_spec["vfs_path"] - final_spec_contents = vfs_files.get(spec_vfs_path) - if final_spec_contents is None: - raise RuntimeError( - f"vfs_result for thread {thread_id} has no file at {spec_vfs_path!r} " - f"(registered as the spec at run start)" - ) - spec_entry = ResumeSpecEntry( - vfs_path=spec_vfs_path, - basename=stored_spec["basename"], - contents=final_spec_contents, - ) + # Reconstruct each registered spec's *final* contents from the completed + # VFS. Missing is a hard error — each was present at registration time. + spec_entries: list[ResumeSpecEntry] = [] + for stored_spec in ri["specs"]: + spec_vfs_path = stored_spec["vfs_path"] + final_spec_contents = vfs_files.get(spec_vfs_path) + if final_spec_contents is None: + raise RuntimeError( + f"vfs_result for thread {thread_id} has no file at {spec_vfs_path!r} " + f"(registered as a spec at run start)" + ) + spec_entries.append(ResumeSpecEntry( + vfs_path=spec_vfs_path, + basename=stored_spec["basename"], + contents=final_spec_contents, + )) return ResumeArtifact( final_intf=_StoredText(path=ra["interface_path"], contents=intf_contents), - spec_entry=spec_entry, + spec_entries=spec_entries, system_doc=_system_handle(ri["system_name"], ri["system"]), commentary=ra["commentary"], intf_path=ra["interface_path"], @@ -439,15 +453,17 @@ async def get_run_info(self, thread_id: str) -> tuple[RunInput, VFSRetriever]: vfs_files = cast(StoredVFS, vfs_item.value)["files"] retriever = VFSRetriever(_files=vfs_files) - stored_spec = ri["spec"] - run_spec: SpecRunEntry = { - "vfs_path": stored_spec["vfs_path"], - "basename": stored_spec["basename"], - "contents": stored_spec["contents"], - } + run_specs: list[SpecRunEntry] = [ + { + "vfs_path": s["vfs_path"], + "basename": s["basename"], + "contents": s["contents"], + } + for s in ri["specs"] + ] run_input: RunInput = { "interface": _StoredText(path=ri["interface_name"], contents=ri["interface_contents"]), - "spec": run_spec, + "specs": run_specs, "system": _system_handle(ri["system_name"], ri["system"]), "reqs": ri["reqs"], } diff --git a/composer/audit/types.py b/composer/audit/types.py index d6b31a9c..8211ddc3 100644 --- a/composer/audit/types.py +++ b/composer/audit/types.py @@ -17,7 +17,7 @@ class RunInput(TypedDict): # Audit-restored file fields are ``Uploadable`` — not renderable content # blocks themselves; the executor rehydrates them through its # ``FileUploader`` into ``Document`` / ``TextDocument`` instances. - spec: SpecRunEntry + specs: list[SpecRunEntry] interface: TextUploadable system: Uploadable reqs: list[str] | None diff --git a/composer/authoring/__init__.py b/composer/authoring/__init__.py new file mode 100644 index 00000000..d976811f --- /dev/null +++ b/composer/authoring/__init__.py @@ -0,0 +1,21 @@ +"""The authoring workflow shared by every formalization backend. + +An authoring session is one stateful agent turn-loop that produces a *spec* — a CVL file, a +foundry test file, a Rust harness — under the same protocol whichever backend it is for: + +* a single ``curr_spec`` buffer the agent writes and edits (:mod:`composer.authoring.buffer`), +* declared skips for properties it will not formalize (:mod:`composer.authoring.state`), +* gate tools that *stamp* a digest of the current buffer into ``validations`` when they pass, +* a feedback judge the agent invokes and can file rebuttals against + (:mod:`composer.authoring.judge`), +* and a publish gate that refuses to finalize until every required stamp matches the buffer + **as it now stands** — so an edit after a green checker run invalidates that run. + +What a backend supplies is the part that genuinely differs: what makes a spec valid at put time, +which tools gate it, and what ground truth the property→check mapping is checked against. Those are +parameters here, not subclasses; the per-backend assembly (which tools, which prompts, which cache) +stays in the backend's own entry point. + +Nothing in this package knows what a CVL rule, a foundry test or a fuzz harness function *is* — +only that each is a *check* carrying a property. +""" diff --git a/composer/authoring/buffer.py b/composer/authoring/buffer.py new file mode 100644 index 00000000..c9c53cde --- /dev/null +++ b/composer/authoring/buffer.py @@ -0,0 +1,223 @@ +"""The ``curr_spec`` buffer and the tools over it. + +One buffer per session, replaced wholesale by a put or surgically by an edit, and read back by +whoever needs the current text (the agent itself, its judge, its checker). Every write goes through +:func:`apply_spec_update`, so a backend that can reject a malformed spec at write time does it in +exactly one place — and a rejected write leaves the buffer untouched. + +Read and edit are :func:`~graphcore.tools.schemas.tool_family` classes, generic in the +backend's state type. A thin factory still binds the tool name, write-time validator, and +display, because those are not schema nouns. +""" + +from dataclasses import dataclass +from typing import Callable, Literal, overload, override +from typing_extensions import TypedDict, ReadOnly + +from langchain_core.messages import AIMessage +from langchain_core.tools import BaseTool +from langgraph.types import Command +from pydantic import Field + +from graphcore.graph import tool_state_update +from graphcore.tools.schemas import ( + ToolFamilyParams, WithAsyncDependencies, WithInjectedId, WithInjectedState, tool_family, +) + +from composer.core.edit import EditErr, EditOk, replace_unique +from composer.ui.tool_display import ToolDisplay, tool_display_of + + +#: Validates a candidate spec at write time. ``None`` accepts it; a string rejects the write and is +#: returned to the agent verbatim, so it must say what is wrong. A backend with no cheap syntactic +#: check passes ``None`` instead of a validator and lets its gate tool be the only judge. +type SpecValidator = Callable[[str], str | None] + +SPEC_KEY = "curr_spec" + +#: The flag a judge's ``did_read`` gate reads. Writing the spec clears it: a review that read the +#: previous draft has not read this one. +READ_KEY = "did_read" + + +class SpecBuffer(TypedDict): + curr_spec: ReadOnly[str | None] + + +class SpecBufferSet(TypedDict): + """A buffer known to be written — a judge's state, which is handed the spec under review.""" + + curr_spec: str + + +class SpecBufferWithRead(SpecBuffer): + did_read: bool + + +def apply_spec_update( + *, + tool_call_id: str, + text: str, + validator: SpecValidator | None = None, + spec_key: str = SPEC_KEY, + reset_read: str | None = None, +) -> str | Command: + """Write ``text`` into the buffer, or reject it. + + Returns the validator's complaint (a plain string, which the agent sees as the tool result and + the buffer is unchanged) or the state update that installs it.""" + if validator is not None and (err := validator(text)) is not None: + return err + update: dict[str, object] = {spec_key: text} + if reset_read: + update[reset_read] = False + return tool_state_update(tool_call_id=tool_call_id, content="Accepted", **update) + + +class BufferDoc(ToolFamilyParams): + description: str + + +@dataclass(frozen=True) +class GetDeps: + missing: str + set_did_read: bool = False + + +@tool_family(BufferDoc) +class GetSpec[T: SpecBuffer]( + WithInjectedId, + WithInjectedState[T], + WithAsyncDependencies[str | Command, GetDeps], +): + """{description}""" + + @override + async def run(self) -> str | Command: + with self.tool_deps() as deps: + spec = self.state[SPEC_KEY] + if spec is None: + return deps.missing + if deps.set_did_read: + return tool_state_update( + tool_call_id=self.tool_call_id, content=spec, **{READ_KEY: True} + ) + return spec + + +@overload +def get_spec_tool[S: SpecBufferWithRead]( + ty: type[S], *, name: str, description: str, missing: str, display: ToolDisplay, + set_did_read: Literal[True], +) -> BaseTool: ... + + +@overload +def get_spec_tool[S: SpecBufferSet]( + ty: type[S], *, name: str, description: str, missing: str, display: ToolDisplay, +) -> BaseTool: ... + + +@overload +def get_spec_tool[S: SpecBuffer]( + ty: type[S], *, name: str, description: str, missing: str, display: ToolDisplay, +) -> BaseTool: ... + + +def get_spec_tool( + ty: type, + *, + name: str, + description: str, + missing: str, + display: ToolDisplay, + set_did_read: bool = False, +) -> BaseTool: + """Read-back tool over the buffer. ``missing`` is what the agent is told when nothing has been + written yet. + + ``set_did_read`` additionally stamps :data:`READ_KEY`, which is how a judge's completion + validator knows the review actually looked at the draft rather than at the copy in its prompt. + """ + return tool_display_of(display)( + GetSpec.with_template(description=description)[ty] + .bind(GetDeps(missing=missing, set_did_read=set_did_read)) + .as_tool(name) + ) + + +@dataclass(frozen=True) +class EditDeps: + name: str + missing: str + validator: SpecValidator | None = None + reset_read: str | None = READ_KEY + + +@tool_family(BufferDoc) +class EditSpec[T: SpecBuffer]( + WithInjectedId, + WithInjectedState[T], + WithAsyncDependencies[str | Command, EditDeps], +): + """{description}""" + old_string: str = Field( + description="The exact span of the current spec to replace. Must occur exactly once; " + "include surrounding context to disambiguate." + ) + new_string: str = Field(description="The text to replace `old_string` with.") + + @override + async def run(self) -> str | Command: + with self.tool_deps() as deps: + spec = self.state[SPEC_KEY] + if spec is None: + return deps.missing + messages = self.state.get("messages") # type: ignore[attr-defined] + if messages: + last = messages[-1] + if ( + isinstance(last, AIMessage) + and len([t for t in last.tool_calls if t["name"] == deps.name]) > 1 + ): + return ( + f"`{deps.name}` tool cannot be called in parallel within the same turn." + ) + match replace_unique(spec, self.old_string, self.new_string): + case EditErr(message=msg): + return msg + case EditOk(text=new_text): + return apply_spec_update( + tool_call_id=self.tool_call_id, + text=new_text, + validator=deps.validator, + reset_read=deps.reset_read, + ) + + +def edit_spec_tool[S: SpecBuffer]( + ty: type[S], + *, + name: str, + description: str, + missing: str, + display: ToolDisplay, + validator: SpecValidator | None = None, + reset_read: str | None = READ_KEY, +) -> BaseTool: + """Surgical single-occurrence replace over the buffer, re-validated exactly as a put is. + + A failed match leaves the buffer alone and returns the reason, which names what to do about it + (add context, or re-read the buffer) — an edit that silently hit the wrong site would be far + worse than one that is refused. + + Two calls of this tool in the same turn are refused: parallel edits race through the state + reducer. A single edit alongside a different tool is allowed. + """ + return tool_display_of(display)( + EditSpec.with_template(description=description)[ty] + .bind(EditDeps( + name=name, missing=missing, validator=validator, reset_read=reset_read, + )) + .as_tool(name) + ) diff --git a/composer/authoring/judge.py b/composer/authoring/judge.py new file mode 100644 index 00000000..1d2e4cfa --- /dev/null +++ b/composer/authoring/judge.py @@ -0,0 +1,294 @@ +"""The feedback judge an authoring session invokes on its own draft. + +A judge is a sub-agent, not a scoring function: it gets the session's tool belt, a rough-draft +scratchpad, a memory namespace of its own, and a read-back of the spec under review, and it must +call ``result`` with a structured :class:`PropertyFeedback`. + +Two properties are enforced rather than requested. It must read the draft back through the tool +(``did_read``) instead of reviewing the copy pasted into its prompt, and the author may file +:class:`RebuttalBase` entries against prior-round feedback, which are rendered into the judge's +input so a point already answered with evidence is answered rather than repeated. +""" + +import inspect +from dataclasses import dataclass +from typing import Any, Awaitable, Callable, NotRequired, Protocol, Sequence + +from langchain_core.tools import BaseTool +from langgraph.graph import MessagesState +from pydantic import BaseModel, Field + +from graphcore.graph import Builder, FlowInput + +from composer.authoring.buffer import SpecBufferSet +from composer.authoring.state import SkippedProperty +from composer.diagnostics.budget import BudgetPressureAbort, pressure_abort_monitor +from composer.spec.context import WorkflowContext +from composer.spec.graph_builder import bind_standard, run_to_completion +from composer.spec.service_host import ServiceHost, Sort +from composer.spec.util import uniq_thread_id +from composer.tools.thinking import RoughDraftState, get_rough_draft_tools + + +class PropertyFeedback(BaseModel): + """ + The feedback on the properties + """ + good: bool = Field(description="Whether the properties are good as is, or if there is room for improvement") + feedback: str = Field(description="The feedback on the rule if work is needed. Can be empty if there is no feedback") + + +# Canned judge verdict when the judge was not run (or was killed mid-run) +# because the author is in its budget wrap-up window. The author's own budget +# warning tells it feedback approval is no longer required, so a dead judge +# is expected there, not an error. +BUDGET_ABORT_FEEDBACK = PropertyFeedback( + good=False, + feedback=( + "The feedback judge was terminated due to budget constraints. " + "See the system alert: feedback approval is no longer required for this task." + ), +) + + +class PropertyFeedbackProtocol(Protocol): + """What a caller of the judge needs from its verdict. A protocol so a backend can hand its + author a richer feedback type without the stamping logic caring.""" + + @property + def good(self) -> bool: ... + + @property + def feedback(self) -> str: ... + + +class RebuttalBase(BaseModel): + prior_feedback_reference: str = Field( + description=( + "A brief quote from, or clear pointer to, the piece of prior-round feedback " + "this rebuttal addresses. Just enough for the judge to identify which prior " + "suggestion you are responding to — not a full transcript." + ) + ) + evidence: str = Field( + description=( + "The concrete artifact backing the rebuttal: typecheck error text, a " + "counterexample summary, a manual quote with location, or a brief reasoned " + "argument. Keep it short and specific — the judge reads this verbatim." + ) + ) + + +#: ``(spec, skipped, rebuttals, within_tool) -> feedback``. ``within_tool`` is the calling feedback +#: tool's ``tool_call_id``, plumbed through to the judge's ``run_to_completion`` so its UI panel +#: anchors under the parent tool widget. +type FeedbackThunk[R: RebuttalBase] = Callable[ + [str, Sequence[SkippedProperty], Sequence[R], str], + Awaitable[PropertyFeedbackProtocol], +] + +#: A :type:`FeedbackThunk` whose every invocation additionally carries a caller-defined context — +#: the editing pipeline's snapshot of the author's working copy — lifted into the judge's input by +#: the ``input_lift`` the judge was built with. +type ContextualFeedbackThunk[R: RebuttalBase, Ctx] = Callable[ + [Ctx, str, Sequence[SkippedProperty], Sequence[R], str], + Awaitable[PropertyFeedbackProtocol], +] + + +class JudgeToolHost(Protocol): + """The judge's construction surface: a builder, the workflow ``sort``, and the tool suite the + judge runs with. Callers vary the FS-read strategy through ``judge_tools`` — frozen fs tools + over the project root, or vfs-aware tools reading the author's working copy — without the judge + machinery knowing which. ``ServiceHost`` consumers adapt via :func:`judge_host_of`.""" + + def builder_heavy(self) -> Builder[None, None, None]: ... + + @property + def sort(self) -> Sort: ... + + @property + def judge_tools(self) -> tuple[BaseTool, ...]: ... + + +@dataclass(frozen=True) +class _ServiceHostJudge: + """The vanilla adapter: judge runs with the host's full tool surface.""" + env: ServiceHost + + def builder_heavy(self) -> Builder[None, None, None]: + return self.env.builder_heavy() + + @property + def sort(self) -> Sort: + return self.env.sort + + @property + def judge_tools(self) -> tuple[BaseTool, ...]: + return self.env.all_tools + + +def judge_host_of(env: ServiceHost) -> JudgeToolHost: + return _ServiceHostJudge(env) + + +class _JudgeExtra(RoughDraftState, SpecBufferSet): + pass + + +class JudgeState(MessagesState, _JudgeExtra): + result: NotRequired[PropertyFeedback] + + +class JudgeInput(FlowInput, _JudgeExtra): + pass + + +def _did_rough_draft_read(s: JudgeState, _: Any) -> str | None: + if not s["did_read"]: + return "Completion REJECTED: never read rough draft for review" + return None + + +#: The judge's builder as its prompt hooks see it: state and input are the judge's own, and no +#: context type is bound. +type JudgeBuilder = Builder[JudgeState, None, JudgeInput] + +#: Applies a prompt to the judge's builder. A backend that renders templates binds its params and +#: calls ``with_*_prompt_template``; one whose prompts are plain strings (a Rust wheel's) calls +#: ``with_sys_prompt`` / ``with_initial_prompt``. Either way the judge itself holds no opinion about +#: where prompt text comes from. +type ApplySystem = Callable[[JudgeBuilder], JudgeBuilder] +type ApplyPrompt[R: RebuttalBase] = Callable[ + [JudgeBuilder, str, Sequence[SkippedProperty], Sequence[R]], JudgeBuilder +] + + +def build_feedback_judge_generic[R: RebuttalBase, S: JudgeState, I: JudgeInput, Ctx]( + *, + st: type[S], + inp: type[I], + ctx: WorkflowContext[Any], + host: JudgeToolHost, + apply_system: Callable[[Builder[S, None, I]], Builder[S, None, I]], + apply_prompt: Callable[ + [Builder[S, None, I], str, Sequence[SkippedProperty], Sequence[R]], Builder[S, None, I] + ], + input_parts: Callable[ + [str, Sequence[SkippedProperty], Sequence[R]], + list[str | dict] | Awaitable[list[str | dict]], + ], + readback: BaseTool, + description: str, + thread_prefix: str, + input_lift: Callable[[JudgeInput, Ctx], I], + extra_tools: Sequence[BaseTool] = (), +) -> ContextualFeedbackThunk[R, Ctx]: + """Compile the judge sub-graph and return the thunk a feedback tool invokes. + + ``apply_prompt`` and ``input_parts`` are the two places a backend decides how the review is + framed: whatever belongs in the prompt itself goes through the first, and whatever is better + said as plain input text goes through the second. Both are called per review and both see the + draft, the skips and the rebuttals, so a prompt that varies with the round can (``input_parts`` + may be async for the same reason — input that reflects state changed between rounds). + + ``st``/``inp`` are the judge's state and input types — :class:`JudgeState`/:class:`JudgeInput` + themselves, or a backend's extensions of them (the editing pipeline's vfs-aware pair). Whatever + the extension adds is seeded per invocation by ``input_lift``, which receives the base input and + the invocation's ``Ctx`` and produces the full ``inp``. + + ``readback`` is the backend's own read-back tool over ``curr_spec``, built with + :func:`composer.authoring.buffer.get_spec_tool` against ``st`` — it keeps the tool + name the backend's prompts already refer to. + + ``ctx`` must be the judge's own context — every backend derives a ``child`` with a ``"judge"`` + key rather than passing the author's. The memory tool is namespaced by the context, and a judge + that shares the author's namespace reviews with the author's notes in hand. + """ + staged = bind_standard( + host.builder_heavy().with_tools(host.judge_tools), + st, + validator=_did_rough_draft_read, + ).with_input( + inp + ).inject( + apply_system + ).with_tools( + [*get_rough_draft_tools(st), ctx.get_memory_tool(), readback, *extra_tools] + ).with_monitor( + pressure_abort_monitor() + ) + + async def judge( + exec_ctx: Ctx, + spec: str, + skipped: Sequence[SkippedProperty], + rebuttals: Sequence[R], + within_tool: str, + ) -> PropertyFeedbackProtocol: + workflow = staged.inject( + lambda b: apply_prompt(b, spec, skipped, rebuttals) + ).compile_async() + produced = input_parts(spec, skipped, rebuttals) + parts = await produced if inspect.isawaitable(produced) else produced + try: + res = await run_to_completion( + workflow, + input_lift( + JudgeInput(input=parts, curr_spec=spec, memory=None, did_read=False), + exec_ctx, + ), + thread_id=uniq_thread_id(thread_prefix), + recursion_limit=ctx.recursion_limit, + description=description, + within_tool=within_tool, + ) + except BudgetPressureAbort: + return BUDGET_ABORT_FEEDBACK + assert "result" in res + return res["result"] + + return judge + + +def build_feedback_judge[R: RebuttalBase]( + *, + ctx: WorkflowContext[Any], + env: ServiceHost, + apply_system: ApplySystem, + apply_prompt: ApplyPrompt[R], + input_parts: Callable[[str, Sequence[SkippedProperty], Sequence[R]], list[str | dict]], + readback: BaseTool, + description: str, + thread_prefix: str, + extra_tools: Sequence[BaseTool] = (), +) -> FeedbackThunk[R]: + """:func:`build_feedback_judge_generic` for the common case: the host's full tool surface, the + base judge state, and no per-invocation context.""" + def lift(i: JudgeInput, _: None) -> JudgeInput: + return i + + judge = build_feedback_judge_generic( + st=JudgeState, + inp=JudgeInput, + ctx=ctx, + host=judge_host_of(env), + apply_system=apply_system, + apply_prompt=apply_prompt, + input_parts=input_parts, + readback=readback, + description=description, + thread_prefix=thread_prefix, + input_lift=lift, + extra_tools=extra_tools, + ) + + async def plain( + spec: str, + skipped: Sequence[SkippedProperty], + rebuttals: Sequence[R], + within_tool: str, + ) -> PropertyFeedbackProtocol: + return await judge(None, spec, skipped, rebuttals, within_tool) + + return plain diff --git a/composer/authoring/state.py b/composer/authoring/state.py new file mode 100644 index 00000000..132087af --- /dev/null +++ b/composer/authoring/state.py @@ -0,0 +1,213 @@ +"""The state an authoring session carries, and the gate that reads it. + +The publish gate is a *digest* gate, not a flag: a checker or judge that accepts the draft stamps +:func:`spec_digest` of the buffer it accepted into ``validations``, and :func:`check_completion` +requires every ``required_validations`` key to carry a stamp equal to the digest of the buffer as it +stands now. Editing the spec after a green run therefore invalidates that run without anything +having to remember to clear it — the stamp simply stops matching. + +The digest covers the skip declarations as well as the spec text, because "this property is skipped, +here is why" is part of what a judge accepted. +""" + +import hashlib +from dataclasses import dataclass +from typing import Annotated, Protocol, Sequence +from typing_extensions import TypedDict + +from pydantic import BaseModel, Field + +from composer.core.state import merge_validation +from composer.spec.types import CheckName, PropertyTitle + + +class SkippedProperty(BaseModel): + """A property the agent explicitly decided not to formalize.""" + property_title: PropertyTitle = Field(description="The unique snake_case title of the property from the batch listing") + reason: str = Field(description="Justification for why this property was skipped") + + +def merge_skips( + left: list[SkippedProperty], + right: list[SkippedProperty], +) -> list[SkippedProperty]: + """State reducer: merge by property_title (new justification replaces old). + + An entry with an empty reason is a sentinel for "unskipped" — it removes + the property from the skip list. + """ + by_title = {s.property_title: s for s in left} + for s in right: + by_title[s.property_title] = s + return sorted( + (s for s in by_title.values() if s.reason), + key=lambda s: s.property_title, + ) + + +def merge_expected_failures( + left: dict[CheckName, str], right: dict[CheckName, str] +) -> dict[CheckName, str]: + """State reducer for the check-name → reason map of checks expected to fail. + + An empty reason removes the marking — the marking tool rejects an empty reason at the tool + boundary, so an empty value can only mean the unmarking tool's delete.""" + to_ret = left.copy() + for k, v in right.items(): + if not v: + to_ret.pop(k, None) + continue + to_ret[k] = v + return to_ret + + +class AuthoringExtra(TypedDict): + """The state every authoring session has, whatever it is authoring. A backend extends this with + its own mapping type and whatever its gate tools record.""" + + curr_spec: str | None + skipped: Annotated[list[SkippedProperty], merge_skips] + validations: Annotated[dict[str, str], merge_validation] + required_validations: list[str] + + +def spec_digest( + curr_spec: str, + skipped: list[SkippedProperty], + version_history: Sequence[str] = (), +) -> str: + """The publish surface's identity: the buffered spec, the skip declarations, and — in the + editing-enabled source pipeline — the applied-edit history, so a stamp earned before a source + edit goes stale with it. Stamps from gate tools are this value, so any later edit to any of the + three invalidates them. Sessions without source editing pass no history and hash identically.""" + digester = hashlib.md5() + digester.update(curr_spec.encode()) + for s in skipped: + digester.update(f"{s.property_title}:{s.reason}".encode()) + for edit_id in version_history: + digester.update(f"edit:{edit_id}".encode()) + return digester.hexdigest() + + +class ValidationStamper(Protocol): + def __call__( + self, state: AuthoringExtra, version_history: Sequence[str] = () + ) -> dict[str, str]: ... + + +def make_validation_stamper(key: str) -> ValidationStamper: + """A ``state -> {key: digest}`` a gate tool merges into ``validations`` when it accepts the + draft. Returned rather than inlined so the stamping tool never spells the digest itself.""" + def stamp(state: AuthoringExtra, version_history: Sequence[str] = ()) -> dict[str, str]: + return {key: spec_digest(state["curr_spec"] or "", state["skipped"], version_history)} + return stamp + + +def check_completion( + state: AuthoringExtra, + version_history: Sequence[str] = (), + *, + nothing_written: str = "no spec written yet.", +) -> str | None: + """None if the publish gate is satisfied, else the reason it is not. + + A stamp that doesn't match the current digest is stale (the agent edited the spec — or, when a + ``version_history`` is in play, the source — after the stamp was issued) and is reported the + same way as a missing one — from the gate's point of view they are the same thing: nothing has + accepted *this* draft.""" + spec = state["curr_spec"] + if spec is None: + return f"Completion REJECTED: {nothing_written}" + digest = spec_digest(spec, state["skipped"], version_history) + validations = state["validations"] + for key in state["required_validations"]: + if validations.get(key) != digest: + return f"Completion REJECTED: {key} validation not satisfied or stale." + return None + + +@dataclass(frozen=True) +class MappingVocab: + """How one backend words the property→check mapping it validates at publish time. Only wording: + the checks themselves are the same everywhere.""" + + #: What one check is called to the agent — "rule", "test", "check". Per-backend because the + #: model does better with its own domain's word than with the framework's generic one. + check_noun: str + #: The name of the publish tool's mapping argument, so a rejection names the field to fix. + field_name: str + #: Where ground truth came from, for the message that rejects a check that never ran. Only read + #: when :func:`validate_check_mapping` is given a ``ran`` set. + ran_source: str = "" + + +def validate_check_mapping( + mapping: Sequence[tuple[PropertyTitle, Sequence[CheckName]]], + skipped: list[SkippedProperty], + titles: Sequence[PropertyTitle], + vocab: MappingVocab, + *, + ran: Sequence[CheckName] | None = None, +) -> str | None: + """Validate the property→checks mapping declared at completion time. None if valid, otherwise + one message enumerating every problem. + + Always checked: every non-skipped property is mapped to at least one non-empty check name, no + skipped property is mapped, every referenced title is one of the batch's, and no title appears + twice. + + ``ran`` is the set of check names the gating run actually executed, when the backend's checker + reports them (forge names every test it ran; a backend whose checker does not is passed + ``None``). Given it, the mapping is checked against ground truth in *both* directions — no + claimed check that didn't run, no check that ran without being tied back to a property — which + is what stops the agent from mapping a property to a check it never wrote. + """ + valid_titles = set(titles) + skipped_titles = {s.property_title for s in skipped} + ran_names = set(ran) if ran is not None else None + noun = vocab.check_noun + errors: list[str] = [] + mapped: set[PropertyTitle] = set() + claimed: set[CheckName] = set() + for title, names_declared in mapping: + if title not in valid_titles: + errors.append(f"Unknown property title {title!r} (not one of the batch's properties).") + continue + if title in mapped: + errors.append(f"Property {title!r} appears more than once in the mapping.") + continue + mapped.add(title) + if title in skipped_titles: + errors.append( + f"Property {title!r} is marked as skipped and must not appear " + "in the mapping (un-skip it or remove it)." + ) + continue + names = [CheckName(n.strip()) for n in names_declared if n.strip()] + if not names: + errors.append(f"Property {title!r} must map to at least one non-empty {noun} name.") + continue + if ran_names is None: + continue + for name in names: + claimed.add(name) + if name not in ran_names: + errors.append( + f"Property {title!r} claims {noun} {name!r}, but no {noun} by that " + f"name ran in {vocab.ran_source}." + ) + for title in titles: + if title in skipped_titles or title in mapped: + continue + errors.append(f"Property {title!r} is neither skipped nor mapped to any {noun}s.") + for name in sorted((ran_names or set()) - claimed): + errors.append( + f"{noun.capitalize()} {name!r} ran but is not tied back to any property in the " + f"mapping. Every {noun} in the file must demonstrate one of the batch's properties." + ) + if errors: + return ( + f"Completion REJECTED: the {vocab.field_name} mapping is invalid. Fix all of the " + "following and resubmit:\n- " + "\n- ".join(errors) + ) + return None diff --git a/composer/authoring/tools.py b/composer/authoring/tools.py new file mode 100644 index 00000000..ebdd77b8 --- /dev/null +++ b/composer/authoring/tools.py @@ -0,0 +1,182 @@ +"""Session tools that are the same for every backend. + +The give-up exit and the skip declarations. Both are worth sharing less for their bodies than for +their *contracts*, which the session reads back: ``failed=True`` with the reason in ``result`` is +how a session reports that it produced nothing, and a skip is what excuses a property from the +publish-time mapping check. Backends agreeing on those by coincidence is one refactor away from two +of them disagreeing. + +What varies on skip and give-up is LLM-facing text, so those are +:func:`~graphcore.tools.schemas.tool_family` classes; each backend supplies its own wording at +instantiate time. Unskip does not vary. +""" + +from typing import Callable, Sequence, override + +from langchain_core.messages import ToolMessage +from langchain_core.tools import BaseTool +from langgraph.types import Command +from pydantic import Field + +from graphcore.graph import tool_state_update +from graphcore.tools.schemas import ( + ToolFamilyParams, WithAsyncDependencies, WithImplementation, WithInjectedId, tool_family, +) + +from composer.authoring.state import SkippedProperty +from composer.spec.types import PropertyTitle +from composer.ui.tool_display import suppress_ack, tool_display, tool_family_display + + +#: Supplies the batch's property titles when a skip tool runs. A thunk rather than the list itself +#: because a backend may only be able to reach them through the graph's runtime context. +type Titles = Callable[[], Sequence[PropertyTitle]] + + +def _as_titles(titles: Titles | Sequence[PropertyTitle]) -> Titles: + if callable(titles): + return titles + snapshot = titles + return lambda: snapshot + + +# --------------------------------------------------------------------------- +# Skip / unskip +# --------------------------------------------------------------------------- + +class SkipParams(ToolFamilyParams): + description: str + reason: str + + +def _skip_label(p: dict, *, description: str, reason: str) -> str: + return f"Skipping property `{p.get('property_title', '?')}`" + + +def _skip_result( + _name: str, msg: ToolMessage, *, description: str, reason: str, +) -> str | None: + return suppress_ack("Skip result", ("Recorded skip",))(_name, msg) + + +@tool_family_display(_skip_label, _skip_result) +@tool_family(SkipParams) +class RecordSkip(WithInjectedId, WithAsyncDependencies[Command, Titles]): + """{description}""" + property_title: PropertyTitle = Field( + description="The snake_case title of the property from the batch listing" + ) + reason: str = Field(description="{reason}") + + @override + async def run(self) -> Command: + with self.tool_deps() as titles: + known = titles() + if self.property_title not in known: + return tool_state_update( + self.tool_call_id, + f"Unknown property title {self.property_title!r}. Must be one " + f"of: {', '.join(known)}.", + ) + if not self.reason.strip(): + return tool_state_update( + self.tool_call_id, + "A non-empty justification is required when skipping a property.", + ) + return tool_state_update( + self.tool_call_id, + f"Recorded skip for property {self.property_title}.", + skipped=[SkippedProperty(property_title=self.property_title, reason=self.reason)], + ) + + +@tool_display( + lambda p: f"Un-skipping property `{p.get('property_title', '?')}`", + suppress_ack("Unskip result", ("Removed skip",)), +) +class Unskip(WithInjectedId, WithAsyncDependencies[Command, Titles]): + """Remove a previously declared skip for a property. Use this if you later find a way to formalize a property you previously skipped.""" + property_title: PropertyTitle = Field( + description="The snake_case title of the property to un-skip" + ) + + @override + async def run(self) -> Command: + with self.tool_deps() as titles: + known = titles() + if self.property_title not in known: + return tool_state_update( + self.tool_call_id, + f"Unknown property title {self.property_title!r}. Must be one " + f"of: {', '.join(known)}.", + ) + return tool_state_update( + self.tool_call_id, + f"Removed skip for property {self.property_title}.", + skipped=[SkippedProperty(property_title=self.property_title, reason="")], + ) + + +def skip_tools( + titles: Titles | Sequence[PropertyTitle], + *, + skip_description: str, + skip_reason: str, +) -> list[BaseTool]: + """The skip / unskip pair, bound to the batch's property titles.""" + get = _as_titles(titles) + return [ + RecordSkip.with_template(description=skip_description, reason=skip_reason) + .bind(get) + .as_tool("record_skip"), + Unskip.bind(get).as_tool("unskip_property"), + ] + + +# --------------------------------------------------------------------------- +# Give up +# --------------------------------------------------------------------------- + +class GiveUpParams(ToolFamilyParams): + description: str + reason: str + label: str + + +def _give_up_label(p: dict, *, description: str, reason: str, label: str) -> str: + return f"Giving up on {label}: {p['reason']}" + + +@tool_family_display(_give_up_label, None) +@tool_family(GiveUpParams) +class GiveUp(WithImplementation[Command], WithInjectedId): + """{description}""" + reason: str = Field(description="{reason}") + + @override + def run(self) -> Command: + return tool_state_update( + self.tool_call_id, + "Accepted", + failed=True, + result=self.reason, + ) + + +def give_up_tool( + *, + name: str, + description: str, + label: str, + reason_description: str = "The reason for giving up on your task", +) -> BaseTool: + """The last-resort exit. ``label`` names the task in the UI line, which reads + ``Giving up on {label}: {reason}``. + + A session that gives up is a real outcome, not an error: the reason is the agent's own account + of why the task could not be completed, and it reaches the report.""" + return GiveUp.with_template( + description=description, + reason=reason_description, + label=label, + ).as_tool(name) diff --git a/composer/cli/cache_autoprove.py b/composer/cli/cache_autoprove.py index c784c67b..f09bb8b7 100644 --- a/composer/cli/cache_autoprove.py +++ b/composer/cli/cache_autoprove.py @@ -7,7 +7,7 @@ Usage:: # by run id (recommended — recovers the cache root, memory namespace, - # threat-model digest and plugin manifest from the run's ``cache_root`` + # threat-model / extra-context digests and plugin manifest from the run's ``cache_root`` # tags; works even when the design doc was auto-discovered): cache-autoprove run @@ -15,7 +15,7 @@ # the design doc, so it does NOT work for auto-discovered runs): cache-autoprove inputs \\ --cache-ns [--memory-ns ] [--threat-model ] \\ - [--plugins ...] + [--extra-context ]... [--plugins ...] """ import argparse @@ -29,40 +29,41 @@ from langgraph.store.base import BaseStore from composer.input.types import DEFAULT_RECURSION_LIMIT -from composer.input.files import file_digest +from composer.input.files import file_digest, resolve_document_paths from composer.ui.cache_explorer import ( CacheNode, StoreNode, CacheTreeNode, CacheExplorerApp, DummyServices, node, section, node_for, leaf, memory, collect_tree, ) from composer.spec.context import WorkflowContext, CVLGeneration, CVLJudge, CacheKey from composer.spec.source.harness import ( - config_key, - system_setup_key, - harness_generation_key, - HARNESS_ANALYSIS_KEY, ContractSetup, SystemDescriptionHarnessed, AgentSystemDescription, HarnessResult, ) from composer.pipeline.cli import root_cache_key, user_ns -from composer.pipeline.core import ( - COMMON_SYSTEM_CACHE_KEY, PROPERTIES_KEY, - _component_cache_key, _batch_cache_key, _pre_property_cache_key, +from composer.pipeline.keys import ( + AGENT_RESULT_KEY, AGENT_ROUND_KEY, BUG_ANALYSIS_KEY, + COMMON_SYSTEM_CACHE_KEY, COMPONENT_KEY, FORMALIZATION_KEY, + PRE_PROPERTY_KEY, PROPERTIES_KEY, SYSTEM_ANALYSIS_KEY, ) from composer.pipeline.plugins import applicable_plugin_manifest, manifest_digest from composer.pipeline.run_tags import AutoProveCacheTags, CACHE_ROOT_RECORD from composer.core.user import get_uid from composer.workflow.services import store_context from composer.io.run_index import get_run_data -from composer.spec.source.summarizer import _summary_key, _SummaryCache -from composer.spec.source.struct_invariant import STRUCTURAL_INV_KEY, Invariants -from composer.spec.source.pipeline import INV_CVL_KEY, AP_PROPERTIES_KEY_NAME +from composer.spec.util import combine_digests +from composer.spec.source.keys import ( + AP_PROPERTIES_KEY_NAME, CVL_JUDGE_KEY, HARNESS_ANALYSIS_KEY, + HARNESS_GENERATION_KEY, INV_CVL_KEY, LAST_ATTEMPT_KEY, STRUCTURAL_INV_KEY, + SUMMARY_KEY, SYSTEM_SETUP_KEY, config_key, +) +from composer.spec.source.summarizer import _SummaryCache +from composer.spec.source.struct_invariant import Invariants from composer.spec.prop_inference import ( _BugAnalysisCache, _AgentResult, _AgentRoundWithHistory, - bug_analysis_key_from_digest, agent_round_key, AGENT_RESULT_KEY, ) -from composer.spec.cvl_generation import GeneratedCVL, _LastAttemptCache, LAST_ATTEMPT_KEY, CVL_JUDGE_KEY +from composer.spec.cvl_generation import GeneratedCVL, _LastAttemptCache from composer.spec.system_model import ( SourceApplication, SourceExplicitContract, SourceExternalActor, HarnessedApplication, HarnessedExplicitContract, HarnessDefinition, @@ -70,11 +71,6 @@ ) -# The driver writes the analyzed SourceApplication under CacheKey(COMMON_SYSTEM_CACHE_KEY) -# (pipeline.core.run_pipeline); mirror that key here to read it back. -SYSTEM_ANALYSIS_KEY = CacheKey[None, SourceApplication](COMMON_SYSTEM_CACHE_KEY) - - # --------------------------------------------------------------------------- # Cache value type # --------------------------------------------------------------------------- @@ -174,19 +170,25 @@ async def _resolve_bug_key( feat_ctx: WorkflowContext, tags: AutoProveCacheTags, ) -> CacheKey: - """The bug-analysis key is parameterized on the threat-model digest and - the refinement (interactive) flag. Both come from the run tags; records + """The bug-analysis key is parameterized on the threat-model and extra-context + digests and the refinement (interactive) flag. All come from the run tags; records written before the ``interactive`` tag existed leave it ``None``, in which case probe both variants and use whichever was written.""" + xc_digest = combine_digests(tags.extra_context_digests) if tags.interactive is not None: - return bug_analysis_key_from_digest( + return BUG_ANALYSIS_KEY( tags.threat_model_digest, with_refinement=tags.interactive, + extra_context_digest=xc_digest, ) for refine in (False, True): - candidate = bug_analysis_key_from_digest(tags.threat_model_digest, refine) + candidate = BUG_ANALYSIS_KEY( + tags.threat_model_digest, refine, extra_context_digest=xc_digest, + ) if await feat_ctx.child(candidate).cache_get(_BugAnalysisCache) is not None: return candidate - return bug_analysis_key_from_digest(tags.threat_model_digest, with_refinement=False) + return BUG_ANALYSIS_KEY( + tags.threat_model_digest, with_refinement=False, extra_context_digest=xc_digest, + ) async def _build_cvl_gen_nodes( @@ -202,13 +204,13 @@ async def _build_component_nodes( store: BaseStore, tags: AutoProveCacheTags, ) -> AsyncGenerator[CacheTreeNode[AutoProveCachedValue], None]: - comp_key = _component_cache_key(feat, manifest_digest(tags.plugins)) + comp_key = COMPONENT_KEY(feat, manifest_digest(tags.plugins)) async with node_for(prop_ctx, comp_key, feat.component.name) as feat_ctx: # Per-plugin pre-inference namespaces are siblings of the component # key under PROPERTIES_KEY, but they're per-component work — surface # them inside the component's subtree. for plugin in tags.plugins: - pre_ctx = prop_ctx.child(_pre_property_cache_key(feat, plugin)) + pre_ctx = prop_ctx.child(PRE_PROPERTY_KEY(feat, plugin)) with node(CacheNode(label=f"Plugin pre-inference: {plugin}", ctx=pre_ctx)): async for n in _enumerate_raw_subtree(store, pre_ctx): yield n @@ -222,7 +224,7 @@ async def _build_component_nodes( i = 0 while True: round_node = await leaf( - agent_ctx, agent_round_key(i), + agent_ctx, AGENT_ROUND_KEY(i), f"Round {i + 1}", _AgentRoundWithHistory, ) if round_node.value is None: @@ -233,7 +235,7 @@ async def _build_component_nodes( bug_cache = await feat_ctx.child(bug_key).cache_get(_BugAnalysisCache) if bug_cache is None: return - batch_key = _batch_cache_key(bug_cache.items) + batch_key = FORMALIZATION_KEY(GeneratedCVL, bug_cache.items) async with node_for(feat_ctx, batch_key, "CVL Generation", GeneratedCVL) as cvl_ctx: async for n in _build_cvl_gen_nodes(cvl_ctx.abstract(CVLGeneration)): yield n @@ -244,7 +246,11 @@ async def build_tree_inner( store: BaseStore, tags: AutoProveCacheTags, ) -> AsyncGenerator[CacheTreeNode[AutoProveCachedValue], None]: - sa_leaf = await leaf(root_ctx, SYSTEM_ANALYSIS_KEY, "system-analysis", SourceApplication) + sa_leaf = await leaf( + root_ctx, + SYSTEM_ANALYSIS_KEY(SourceApplication, COMMON_SYSTEM_CACHE_KEY), + "system-analysis", SourceApplication, + ) yield sa_leaf # Read config value upfront so we can derive the summary key outside the with block @@ -252,20 +258,20 @@ async def build_tree_inner( async with node_for(root_ctx, config_key, "config", ContractSetup) as config_ctx: if sa_leaf.value is not None: - async with node_for(config_ctx, system_setup_key(sa_leaf.value), "setup", SystemDescriptionHarnessed) as setup_ctx: + async with node_for(config_ctx, SYSTEM_SETUP_KEY(sa_leaf.value), "setup", SystemDescriptionHarnessed) as setup_ctx: ha_leaf = await leaf(setup_ctx, HARNESS_ANALYSIS_KEY, "harness-analysis", AgentSystemDescription) yield ha_leaf if ha_leaf.value is not None and ha_leaf.value.needs_harnessing(): yield await leaf( setup_ctx, - harness_generation_key(ha_leaf.value), + HARNESS_GENERATION_KEY(ha_leaf.value), "harness-generation", HarnessResult, ) # Summary — key derivable only once ContractSetup is cached if config_val is not None: - yield await leaf(root_ctx, _summary_key(config_val), "summary", _SummaryCache) + yield await leaf(root_ctx, SUMMARY_KEY(config_val), "summary", _SummaryCache) yield await leaf(root_ctx, STRUCTURAL_INV_KEY, "structural-inv", Invariants) async with node_for(root_ctx, INV_CVL_KEY, "invariant-cvl", GeneratedCVL) as inv_cvl_ctx: @@ -481,6 +487,11 @@ def _resolve_from_inputs(args: argparse.Namespace) -> AutoProveCacheTags | None: file_digest(pathlib.Path(args.threat_model)) if args.threat_model is not None else None ) + # The run's own resolver, so the order (hence the key) is reproduced exactly. + xc_digests = [ + file_digest(p) + for p in resolve_document_paths(args.extra_context) + ] return AutoProveCacheTags( cache_root=list(root_ns), @@ -488,6 +499,7 @@ def _resolve_from_inputs(args: argparse.Namespace) -> AutoProveCacheTags | None: memory_ns=memory_ns, plugins=plugins, threat_model_digest=tm_digest, + extra_context_digests=xc_digests, # Not recoverable from the inputs — the tree builder probes both # refinement variants of the bug-analysis key. interactive=None, @@ -546,6 +558,8 @@ async def _async_main(args: argparse.Namespace) -> int: status += f" | Plugins: {', '.join(tags.plugins)}" if tags.threat_model_digest: status += f" | TM digest: {tags.threat_model_digest}" + if tags.extra_context_digests: + status += f" | XC digests: {', '.join(tags.extra_context_digests)}" app = CacheExplorerApp( build_tree=lambda: build_tree(root_ctx, store, tags), @@ -566,7 +580,8 @@ def main() -> int: p_run = sub.add_parser( "run", help="Explore a run by id (recommended). Reads the tags the run recorded " - "in its metadata (cache root, memory ns, plugins, threat-model digest) " + "in its metadata (cache root, memory ns, plugins, threat-model and " + "extra-context digests) " "— works even when the design doc was auto-discovered.", ) p_run.add_argument("run_id", help="Run id (from the autoprove logs / ap-trail).") @@ -591,6 +606,12 @@ def main() -> int: help="Path to the threat model used for the original run — its " "digest parameterizes the bug-analysis cache key. Omit for " "runs without one.") + p_inputs.add_argument("--extra-context", dest="extra_context", action="append", + default=None, metavar="PATH", + help="Extra-context document (or directory of them) used for the " + "original run — their combined digest also parameterizes the " + "bug-analysis cache key. Repeat in the original order; a swept " + "directory must still hold the same files.") p_inputs.add_argument("--plugins", dest="plugins", nargs="*", default=None, help="Plugin names active for the original run — the manifest " "digest is suffixed onto per-component cache keys. Defaults " diff --git a/composer/cli/cache_natspec.py b/composer/cli/cache_natspec.py index 94a54b28..f11a417c 100644 --- a/composer/cli/cache_natspec.py +++ b/composer/cli/cache_natspec.py @@ -42,7 +42,7 @@ ) from composer.spec.prop_inference import ( _BugAnalysisCache, _AgentResult, _AgentRoundWithHistory, - bug_analysis_key, AGENT_RESULT_KEY, agent_round_key, + BUG_ANALYSIS_KEY, AGENT_RESULT_KEY, AGENT_ROUND_KEY, ) from composer.spec.cvl_generation import ( _LastAttemptCache, CVL_JUDGE_KEY, LAST_ATTEMPT_KEY, @@ -145,7 +145,7 @@ async def build_component_tree( async with node_for(contract_ctx, key, comp.name) as feat_ctx: # Bug analysis cache: aggregate (_BugAnalysisCache.items) → agent # result (_AgentResult) → per-round (_AgentRoundWithHistory). - bug_key = bug_analysis_key(None, with_refinement=with_refinement) + bug_key = BUG_ANALYSIS_KEY(None, with_refinement=with_refinement) async with node_for(feat_ctx, bug_key, "Bug Analysis", _BugAnalysisCache) as bug_ctx: async with node_for( bug_ctx, AGENT_RESULT_KEY, "Agent result", _AgentResult, @@ -155,7 +155,7 @@ async def build_component_tree( i = 0 while True: round_node = await leaf( - agent_ctx, agent_round_key(i), + agent_ctx, AGENT_ROUND_KEY(i), f"Round {i + 1}", _AgentRoundWithHistory, ) if round_node.value is None: diff --git a/composer/cli/tui_autoprove.py b/composer/cli/tui_autoprove.py index 83ed5d4c..250dd017 100644 --- a/composer/cli/tui_autoprove.py +++ b/composer/cli/tui_autoprove.py @@ -1,6 +1,7 @@ """Entry point for the auto-prove multi-agent pipeline TUI.""" import asyncio +import logging import composer.bind as _ @@ -8,6 +9,8 @@ from composer.ui.autoprove_app import AutoProveApp from composer.spec.source.autoprove_common import _entry_point +_log = logging.getLogger(__name__) + # --------------------------------------------------------------------------- # Main # --------------------------------------------------------------------------- @@ -27,10 +30,12 @@ async def work(): if result.failures: msg += f", {len(result.failures)} failures" app.notify(msg) - app._pipeline_done = True + app.mark_pipeline_done() except Exception as exc: + # A toast alone loses the failure the moment it fades — and the traceback with it. + _log.exception("pipeline failed") app.notify(f"Pipeline failed: {exc}", severity="error") - app._pipeline_done = True + app.mark_pipeline_done() app.set_work(work) await app.run_async() diff --git a/composer/cli/tui_foundry.py b/composer/cli/tui_foundry.py index 80f915e6..230d95a7 100644 --- a/composer/cli/tui_foundry.py +++ b/composer/cli/tui_foundry.py @@ -1,6 +1,7 @@ """Entry point for the foundry test-generation pipeline TUI.""" import asyncio +import logging from typing import cast import pathlib @@ -12,6 +13,8 @@ from composer.ui.foundry_app import FoundryApp from composer.pipeline.ptypes import Delivered +_log = logging.getLogger(__name__) + # --------------------------------------------------------------------------- # Main # --------------------------------------------------------------------------- @@ -38,10 +41,12 @@ async def work(): if result.failures: msg += f", {len(result.failures)} failures" app.notify(msg) - app._pipeline_done = True + app.mark_pipeline_done() except Exception as exc: + # A toast alone loses the failure the moment it fades — and the traceback with it. + _log.exception("pipeline failed") app.notify(f"Pipeline failed: {exc}", severity="error") - app._pipeline_done = True + app.mark_pipeline_done() app.set_work(work) await app.run_async() diff --git a/composer/cli/tui_pipeline.py b/composer/cli/tui_pipeline.py index b9bda3b6..ac662e7e 100644 --- a/composer/cli/tui_pipeline.py +++ b/composer/cli/tui_pipeline.py @@ -9,6 +9,7 @@ import argparse import asyncio +import logging import json import pathlib import sys @@ -41,6 +42,8 @@ from composer.ui.pipeline_app import NatspecPipelineApp from composer.cli.natspec_startup import build_mental_model, make_source_factory +_log = logging.getLogger(__name__) + # --------------------------------------------------------------------------- # Args @@ -234,9 +237,11 @@ async def work(): ) await app.on_pipeline_done(result) except Exception as exc: + # A toast alone loses the failure the moment it fades — and the traceback with it. + _log.exception("pipeline failed") app.notify(f"Pipeline failed: {exc}", severity="error") await app.mount_error(exc) - app._pipeline_done = True + app.mark_pipeline_done() app.set_work(work) await app.run_async() diff --git a/composer/core/context.py b/composer/core/context.py index ac07f816..a53437ba 100644 --- a/composer/core/context.py +++ b/composer/core/context.py @@ -4,7 +4,7 @@ from graphcore.tools.vfs import VFSAccessor from composer.core.state import AIComposerState -from composer.core.validation import ValidationType, prover +from composer.core.validation import CodegenValidation from composer.prover.core import DEFAULT_GLOBAL_TIMEOUT @dataclass @@ -30,11 +30,21 @@ class AIComposerContext: # (CEX handler, prover options) ride ``ProverDeps`` on the prover tool, and # ``rag_db`` is injected directly into the CVL tools — neither belongs here. vfs_materializer: VFSAccessor[AIComposerState] - required_validations: list[ValidationType] = field(default_factory=lambda: [prover]) + required_validations: list[CodegenValidation] -def compute_state_digest(c: AIComposerContext, state: AIComposerState) -> str: - # not interested in cryptographic bulletproofing, just need *some* digest +def compute_state_digest(state: AIComposerState) -> str: + # Digest the VFS overlay only — the agent-authored / dirty files. NOT the + # materialized tree: a source-root run's fs_layer underlay (OZ deps, etc.) + # is immutable for the run, so re-hashing it on every prover stamp is pure + # waste. The state a validation stamp cares about lives in the VFS overlay. digester = hashlib.md5() - for (_, cont) in sorted(c.vfs_materializer.iterate(state), key = lambda x: x[0]): - digester.update(cont) + for (_, cont) in sorted(state["vfs"].items(), key=lambda x: x[0]): + digester.update(cont.encode("utf-8")) return digester.hexdigest() + +def stamp(req: CodegenValidation, st: AIComposerState) -> dict: + return { + "validation": { + req.to_key(): compute_state_digest(st) + } + } \ No newline at end of file diff --git a/composer/core/edit.py b/composer/core/edit.py index 921413e3..e0d884a8 100644 --- a/composer/core/edit.py +++ b/composer/core/edit.py @@ -30,7 +30,7 @@ class EditErr: message: str -def replace_unique(buffer: str, old: str, new: str) -> EditOk | EditErr: +def replace_unique(buffer: str, old: str, new: str, replace_all: bool = False) -> EditOk | EditErr: """Replace the single occurrence of ``old`` in ``buffer`` with ``new``. An edit must identify exactly one site. Returns :class:`EditErr` when @@ -38,16 +38,20 @@ def replace_unique(buffer: str, old: str, new: str) -> EditOk | EditErr: surrounding context) rather than risk silently editing the wrong place. On success returns :class:`EditOk` with the rewritten buffer. """ - count = buffer.count(old) - if count == 0: - return EditErr( - "`old_string` was not found in the current buffer. It must match an " - "exact span of the contents, including whitespace and indentation. " - "Read the buffer back and copy the target span verbatim." - ) - if count > 1: - return EditErr( - f"`old_string` matched {count} locations; it must match exactly one. " - "Include more surrounding context so the target span is unique." - ) - return EditOk(buffer.replace(old, new, 1)) + if not replace_all: + count = buffer.count(old) + if count == 0: + return EditErr( + "`old_string` was not found in the current buffer. It must match an " + "exact span of the contents, including whitespace and indentation. " + "Read the buffer back and copy the target span verbatim." + ) + if count > 1: + return EditErr( + f"`old_string` matched {count} locations; it must match exactly one. " + "Include more surrounding context so the target span is unique." + ) + replace_count = 1 + else: + replace_count = -1 + return EditOk(buffer.replace(old, new, replace_count)) diff --git a/composer/core/validation.py b/composer/core/validation.py index b3d4056b..750741df 100644 --- a/composer/core/validation.py +++ b/composer/core/validation.py @@ -1,6 +1,29 @@ -from typing import Literal +from dataclasses import dataclass -ValidationType = Literal["prover", "natural language requirements"] +@dataclass(frozen=True) +class ProverValidation: + """Completion gate: the Certora Prover verified a committed spec against the + generated code. One gate per registered spec, keyed by its VFS path.""" -prover : ValidationType = "prover" -reqs : ValidationType = "natural language requirements" + spec: str + + def to_key(self) -> str: + return f"prover:{self.spec}" + + def description(self) -> str: + return f"prover verification of {self.spec}" + + +@dataclass(frozen=True) +class ReqsValidation: + """Completion gate: the implementation satisfies the extracted + natural-language requirements (stamped by the requirements judge).""" + + def to_key(self) -> str: + return "natural language requirements" + + def description(self) -> str: + return "satisfaction of the natural-language requirements" + + +type CodegenValidation = ProverValidation | ReqsValidation diff --git a/composer/cvl/tools.py b/composer/cvl/tools.py index 6899eee8..65c6bba4 100644 --- a/composer/cvl/tools.py +++ b/composer/cvl/tools.py @@ -10,21 +10,22 @@ import subprocess import tempfile from typing import Annotated, Literal, overload -from typing_extensions import TypedDict +from langchain_core.messages import AIMessage +from typing_extensions import TypedDict, ReadOnly from langchain_core.tools import tool, InjectedToolCallId, BaseTool from langgraph.types import Command -from langgraph.prebuilt import InjectedState -from pydantic import BaseModel, Field, create_model +from pydantic import BaseModel, Field +from composer.authoring.buffer import ( + READ_KEY, SPEC_KEY, SpecBuffer, SpecBufferSet, SpecBufferWithRead, + apply_spec_update, edit_spec_tool, get_spec_tool, +) from composer.certora_env import typechecker_jar -from composer.core.edit import replace_unique, EditOk, EditErr from composer.cvl.schema import CVLFile from composer.cvl.pretty_print import pretty_print from composer.ui.tool_display import tool_display_of, CommonTools, ToolDisplay, suppress_ack -from graphcore.graph import tool_state_update - _logger = logging.getLogger(__name__) _put_cvl_display = ToolDisplay( @@ -58,9 +59,9 @@ class PutCVLSchemaLG(BaseModel): PutCVLSchemaLG.__doc__ = put_cvl_description -DEFAULT_READ_KEY = "did_read" +DEFAULT_READ_KEY = READ_KEY -DEFAULT_SPEC_KEY = "curr_spec" +DEFAULT_SPEC_KEY = SPEC_KEY class PutCVLRaw(BaseModel): @@ -75,19 +76,13 @@ class PutCVLRaw(BaseModel): tool_call_id: Annotated[str, InjectedToolCallId] -def maybe_update_cvl( - *, - tool_call_id: str, - pp: str, - spec_key: str, - ast_json: dict | None = None, - reset_read: str | None = None -) -> str | Command: - """ - Validate CVL syntax and update state if valid. +def cvl_syntax_error(pp: str, ast_json: dict | None = None) -> str | None: + """The CVL parser's complaint about ``pp``, or ``None`` if it parses — the + :data:`~composer.authoring.buffer.SpecValidator` for every CVL buffer write. - Uses the Certora emv.jar parser to validate the CVL syntax. - Returns a Command to update state on success, or an error message on failure. + ``ast_json`` is the AST a structured put was rendered from; it is dumped alongside the + pretty-printed text when the parse fails, since a rejection there is a pretty-printer bug + rather than a spec the agent can fix. """ # Resolve the typechecker jar and run it. A failure in either step is an # environment/plumbing problem (jar not packaged, CERTORA misconfigured, java @@ -128,14 +123,29 @@ def maybe_update_cvl( stderr: {res.stderr} """ - update = {} - update[spec_key] = pp - if reset_read: - update[reset_read] = False - return tool_state_update( + return None + + +def maybe_update_cvl( + *, + tool_call_id: str, + pp: str, + spec_key: str, + ast_json: dict | None = None, + reset_read: str | None = None +) -> str | Command: + """ + Validate CVL syntax and update state if valid. + + Uses the Certora emv.jar parser to validate the CVL syntax. + Returns a Command to update state on success, or an error message on failure. + """ + return apply_spec_update( tool_call_id=tool_call_id, - content="Accepted", - **update + text=pp, + spec_key=spec_key, + reset_read=reset_read, + validator=lambda text: cvl_syntax_error(text, ast_json), ) @@ -162,35 +172,30 @@ def put_cvl_raw( """Put a CVL file using raw surface syntax.""" return maybe_update_cvl(tool_call_id=tool_call_id, pp=cvl_file, reset_read=DEFAULT_READ_KEY, spec_key=DEFAULT_SPEC_KEY) -class WithCurrSpec(TypedDict): - curr_spec: str | None +#: The CVL flows' names for the shared buffer state shapes. +WithCurrSpec = SpecBuffer +WithCurrSpecAndDidRead = SpecBufferWithRead +WithCurrSpecNonNull = SpecBufferSet -class WithCurrSpecAndDidRead(WithCurrSpec): - did_read: bool - -class WithCurrSpecNonNull(TypedDict): - curr_spec: str - -class GetCVLSchemaTemplate(BaseModel): - """ +_GET_CVL_DESCRIPTION = """ Retrive the textual representation of the current specification. """ @overload -def get_cvl[S: WithCurrSpecAndDidRead]( +def get_cvl[S: SpecBufferWithRead]( ty: type[S], *, set_did_read: Literal[True], ) -> BaseTool: ... @overload -def get_cvl[S: WithCurrSpecNonNull]( +def get_cvl[S: SpecBufferSet]( ty: type[S], ) -> BaseTool: ... @overload -def get_cvl[S: WithCurrSpec]( +def get_cvl[S: SpecBuffer]( ty: type[S], ) -> BaseTool: ... @@ -199,36 +204,24 @@ def get_cvl( *, set_did_read: bool = False, ) -> BaseTool: - extra_fields: dict = {} + """The CVL read-back tool over ``curr_spec``. ``set_did_read`` additionally stamps + ``did_read``, which the property judge's completion validator requires.""" if set_did_read: - extra_fields["tool_call_id"] = (Annotated[str, InjectedToolCallId], ...) - schema = create_model( - "GetCVL", - __base__=GetCVLSchemaTemplate, - __doc__=GetCVLSchemaTemplate.__doc__, - state=(Annotated[ty, InjectedState], ...), - **extra_fields, + return get_spec_tool( + ty, + name="get_cvl", + description=_GET_CVL_DESCRIPTION, + missing="No spec file written yet", + display=_get_cvl_display, + set_did_read=True, + ) + return get_spec_tool( + ty, + name="get_cvl", + description=_GET_CVL_DESCRIPTION, + missing="No spec file written yet", + display=_get_cvl_display, ) - @tool_display_of(_get_cvl_display) - @tool(args_schema=schema) - def get_cvl( - **args - ) -> str | Command: - st = args["state"] - if st["curr_spec"] is None: - return "No spec file written yet" - spec = st["curr_spec"] - if set_did_read: - update = { - DEFAULT_READ_KEY: True - } - return tool_state_update( - tool_call_id=args["tool_call_id"], - content=spec, - **update - ) - return spec - return get_cvl edit_cvl_description = """ @@ -242,42 +235,20 @@ def get_cvl( The edited spec is run through the CVL parser exactly like `put_cvl_raw`. If the result fails to parse, the edit is rejected with the parser errors and the buffer is unchanged. -""" - -class EditCVLTemplate(BaseModel): - old_string: str = Field( - description="The exact span of the current spec to replace. Must occur exactly once; " - "include surrounding context to disambiguate." - ) - new_string: str = Field(description="The text to replace `old_string` with.") +IMPORTANT: You cannot call this tool multiple times in the same turn. If you need to make +multiple edits, you must spread them across distinct turns. +""" -def edit_cvl[S: WithCurrSpec](ty: type[S]) -> BaseTool: +def edit_cvl[S: SpecBuffer](ty: type[S]) -> BaseTool: """A surgical-edit tool over the ``curr_spec`` buffer: single-occurrence string replace, then re-validate the result exactly like ``put_cvl_raw``.""" - schema = create_model( - "EditCVL", - __base__=EditCVLTemplate, - __doc__=edit_cvl_description, - state=(Annotated[ty, InjectedState], ...), - tool_call_id=(Annotated[str, InjectedToolCallId], ...), + return edit_spec_tool( + ty, + name="edit_cvl", + description=edit_cvl_description, + missing="No spec file written yet — use put_cvl or put_cvl_raw first.", + display=_edit_cvl_display, + validator=cvl_syntax_error, ) - - @tool_display_of(_edit_cvl_display) - @tool(args_schema=schema) - def edit_cvl(**args) -> str | Command: - st = args["state"] - if st["curr_spec"] is None: - return "No spec file written yet — use put_cvl or put_cvl_raw first." - match replace_unique(st["curr_spec"], args["old_string"], args["new_string"]): - case EditErr(message=msg): - return msg - case EditOk(text=new_text): - return maybe_update_cvl( - tool_call_id=args["tool_call_id"], - pp=new_text, - spec_key=DEFAULT_SPEC_KEY, - reset_read=DEFAULT_READ_KEY, - ) - return edit_cvl \ No newline at end of file diff --git a/composer/diagnostics/budget.py b/composer/diagnostics/budget.py new file mode 100644 index 00000000..431bf806 --- /dev/null +++ b/composer/diagnostics/budget.py @@ -0,0 +1,319 @@ +from typing import Iterator, Callable, Any, Mapping, Never, Protocol, Literal, TypedDict, LiteralString +from typing_extensions import TypeVar, ReadOnly +from contextlib import contextmanager +from contextvars import ContextVar +from dataclasses import dataclass +import time + +from langgraph.graph import MessagesState +from graphcore.graph import StateMonitor, MonitorReturn +from langchain_core.messages import HumanMessage, AnyMessage +from .timing import RunSummary, get_run_summary_or_none + +StateVar = TypeVar("StateVar", default=MessagesState, bound=MessagesState) + +# The fraction of a budget at which "budget pressure" begins: budget_monitor's +# warning fires and budget_pressure() flips true. One global so every accessor +# agrees on where the wrap-up window starts. +BUDGET_PRESSURE_THRESHOLD = 0.8 + + +class BudgetExceeded(Exception): + """Hard budget stop, raised cooperatively (from a monitor, between agent + turns) once the active budget is blown. Only monitors given an + ``on_overbudget`` callback raise; the workflow that launched the agent + catches this and converts it into its give-up result.""" + + +class BudgetPressureAbort(Exception): + """Raised by ``pressure_abort_monitor`` to terminate an auxiliary agent + (e.g. a feedback judge) whose output is worthless once the main agent is + in its wrap-up window. Caught by the tool that launched the agent.""" + +type ConstraintType = Literal["time", "token"] + +class _ConstraintWarnings(TypedDict): + time: ReadOnly[str] + token: ReadOnly[str] + +class ResourceConstraint(Protocol): + @property + def sort(self) -> ConstraintType: + ... + + def overbudget(self) -> bool: + ... + + def pressured(self, threshold: float = BUDGET_PRESSURE_THRESHOLD) -> bool: + ... + +@dataclass +class BudgetCounter: + """One node of the caps-over-pool scheme. Leaf counters are per-phase + *caps* whose ``parent`` is the run's shared *pool* (the real budget); + ``token_cost_budget`` creates parentless one-off counters. Cost accrues + up the chain, and both the hard stop and the pressure window trip on + whichever level is tighter — a phase can only starve later phases up to + its cap, while unspent phase money never leaves the pool (rollover is + automatic, not an explicit transfer).""" + total_budget: float + curr_cost: float + parent: "BudgetCounter | None" = None + + def overbudget(self) -> bool: + if self.curr_cost > self.total_budget: + return True + return self.parent.overbudget() if self.parent is not None else False + + def pressured(self, threshold: float = BUDGET_PRESSURE_THRESHOLD) -> bool: + if self.curr_cost >= self.total_budget * threshold: + return True + return self.parent.pressured(threshold) if self.parent is not None else False + + @property + def sort(self) -> ConstraintType: + return "token" + +_budget_accumulator = ContextVar[None | BudgetCounter]("_budget_accumulator", default=None) + +_cost_centers = ContextVar[None | dict[str, BudgetCounter]]("_cost_centers", default=None) + +# The *name* of the innermost named budget scope. Deliberately tracked separately from +# the counters — and unconditionally, budget or no budget — so telemetry (the thread +# logger stamps it into ThreadMeta.cost_center) can attribute work to phases on +# unbudgeted runs too. +_cost_center_name = ContextVar[str | None]("_cost_center_name", default=None) + +_time_budget = ContextVar[float | None]("_time_budget", default=None) + +@dataclass +class _TimeCounter: + budget: float + + summ: RunSummary + + def overbudget(self) -> bool: + return self.summ.total_wall_s() > self.budget + + def pressured(self, threshold: float = BUDGET_PRESSURE_THRESHOLD) -> bool: + return self.summ.total_wall_s() > self.budget * threshold + + @property + def sort(self) -> ConstraintType: + return "time" + + +def _get_constraints() -> list[ResourceConstraint]: + to_ret : list[ResourceConstraint] = [] + if (b := _budget_accumulator.get()) is not None: + to_ret.append(b) + if (t_b := _time_budget.get()) is not None and \ + (rs := get_run_summary_or_none()) is not None: + to_ret.append(_TimeCounter(t_b, rs)) + return to_ret + +def current_cost_center() -> str | None: + """The name of the innermost named budget scope, or ``None`` outside any (the run + pool, or code that runs before/after the pipeline). Valid regardless of whether a + budget is installed.""" + return _cost_center_name.get() + +DEFAULT_RESOURCE_PRESSURE_MESSAGE = """ + +You have almost exceeded the {resource} allotted for this task. + +Finish your task in as orderly a fashion as possible; partial/incomplete results are better +than going over budget. + +""" + +DEFAULT_BUDGET_PRESSURE_MESSAGE = DEFAULT_RESOURCE_PRESSURE_MESSAGE.format(resource="token cost budget") + +DEFAULT_TIME_PRESSURE_MESSAGE = DEFAULT_RESOURCE_PRESSURE_MESSAGE.format(resource="time") + +_DEFAULT_WARNINGS : _ConstraintWarnings = { + "time": DEFAULT_TIME_PRESSURE_MESSAGE, + "token": DEFAULT_BUDGET_PRESSURE_MESSAGE +} + +class _WarningState(TypedDict): + time: bool + token: bool + +@contextmanager +def time_budget( + total: float +) -> Iterator[None]: + prev = _time_budget.get() + if prev is not None: + raise ValueError("Timer already set, nested timings not supported") + prev_tok = _time_budget.set(total) + try: + yield + finally: + _time_budget.reset(prev_tok) + +@contextmanager +def total_budget( + total: float, + caps: Mapping[str, float] +) -> Iterator[None]: + """Install the run's budget: ``total`` is the pool (the real bound on + spend) and ``caps`` are per-phase ceilings. Caps need not sum to the + pool — they only bound how much a single phase may hog, so each can be + generous; whatever a phase doesn't spend simply remains in the pool for + later phases.""" + curr = _cost_centers.get() + if curr is not None: + raise RuntimeError("Budget already installed, cannot overwrite existing.") + pool = BudgetCounter(total_budget=total, curr_cost=0.0) + prev = _cost_centers.set({ + k: BudgetCounter(total_budget=v, curr_cost=0.0, parent=pool) for (k, v) in caps.items() + }) + # Work running outside any named center (e.g. the report phase) accrues + # to — and feels pressure from — the pool directly. + prev_accum = _budget_accumulator.set(pool) + try: + yield + finally: + _budget_accumulator.reset(prev_accum) + _cost_centers.reset(prev) + +@contextmanager +def named_budget( + nm: str +) -> Iterator[None]: + if (res := _cost_centers.get()) is None: + raise RuntimeError("No costs installed") + if nm not in res: + raise RuntimeError(f"Named budget item not known: {nm}") + prev = _budget_accumulator.set(res[nm]) + name_tok = _cost_center_name.set(nm) + try: + yield + finally: + _cost_center_name.reset(name_tok) + _budget_accumulator.reset(prev) + +@contextmanager +def named_budget_or_nop( + nm: str +) -> Iterator[None]: + if (_cost_centers.get()) is None: + # No budget installed: no counters to bind, but still stamp the cost-center + # name — phase attribution is telemetry, not billing. (A @contextmanager + # generator must yield exactly once even on this path.) + name_tok = _cost_center_name.set(nm) + try: + yield + finally: + _cost_center_name.reset(name_tok) + return + with named_budget(nm): + yield + +@contextmanager +def token_cost_budget( + total_cost: float, +) -> Iterator[None]: + if _budget_accumulator.get() is not None: + raise RuntimeError("Nested budgets not supported") + accum = BudgetCounter(total_budget=total_cost, curr_cost=0.0) + prev = _budget_accumulator.set(accum) + try: + yield + finally: + _budget_accumulator.reset(prev) + +def accumulate_cost( + cost: float +): + # Accrue up the chain: the active center and (through parent) the pool. + accum = _budget_accumulator.get() + while accum is not None: + accum.curr_cost += cost + accum = accum.parent + +def _none_if_empty[ + T: list[AnyMessage] | dict[str, Any] +]( + s: T +) -> T | None: + if not s: + return None + return s + +def budget_monitor( + *, + warn_threshold: float = BUDGET_PRESSURE_THRESHOLD, + warning_message: str | Callable[[StateVar, ConstraintType], str] | None = None, + state_transformer: Callable[[StateVar, ConstraintType], dict[str, Any]] | None = None, + on_overbudget: Callable[[ConstraintType], None] | None = None +) -> StateMonitor[StateVar]: + accum = _get_constraints() + if len(accum) == 0: + return lambda _ign: (None, None) + warned : _WarningState = { + "time": False, + "token": False + } + def monitor( + curr_state: StateVar + ) -> MonitorReturn: + to_ret : list[AnyMessage] = [] + upd : dict[str, Any] = {} + for acc in accum: + if acc.overbudget() and on_overbudget is not None: + on_overbudget(acc.sort) + if not acc.pressured(warn_threshold) or warned[acc.sort]: + continue + warned[acc.sort] = True + + if warning_message is None: + msg = _DEFAULT_WARNINGS[acc.sort] + elif isinstance(warning_message, str): + msg = warning_message + else: + msg = warning_message(curr_state, acc.sort) + to_ret.append(HumanMessage(msg)) + if state_transformer is not None: + upd.update(state_transformer(curr_state, acc.sort)) + return _none_if_empty(to_ret), _none_if_empty(upd) + return monitor + +def constraint_sort_to_noun(s: ConstraintType) -> LiteralString: + match s: + case "time": + return "Time" + case "token": + return "Token cost" + +def raise_budget_exceeded(sort: ConstraintType) -> Never: + """``on_overbudget`` callback for agents that opt into the hard stop.""" + raise BudgetExceeded( + f"{constraint_sort_to_noun(sort)} budget exhausted; the agent was cooperatively terminated." + ) + + +def budget_pressure() -> bool: + """Whether the active budget is inside its wrap-up window: accrued cost at + or past ``BUDGET_PRESSURE_THRESHOLD`` of the phase cap *or* of the run + pool, whichever trips first. False when no budget is installed. Use this + to skip launching work that would only be told to immediately pack it in + (e.g. further property-extraction rounds).""" + res = _get_constraints() + return any(r.pressured() for r in res) + + +def pressure_abort_monitor() -> StateMonitor[MessagesState]: + """Monitor for auxiliary agents (feedback judges) that should not outlive + the main agent's wrap-up window: raises ``BudgetPressureAbort`` between + turns once budget pressure sets in. The tool that launched the agent + catches the exception and returns a canned "terminated for budget" + result. Reads the budget at call time, so it can be attached to a graph + compiled outside any budget scope.""" + def monitor(_curr_state: StateVar) -> MonitorReturn: + if budget_pressure(): + raise BudgetPressureAbort() + return (None, None) + return monitor diff --git a/composer/diagnostics/cost_callback.py b/composer/diagnostics/cost_callback.py new file mode 100644 index 00000000..aded9c76 --- /dev/null +++ b/composer/diagnostics/cost_callback.py @@ -0,0 +1,84 @@ +"""LangChain callback that accumulates the running USD cost of an LLM's calls. + +Sibling to :class:`composer.diagnostics.usage_callback.UsageCallback`: attached at +model construction so it fires for *every* ``invoke`` / ``ainvoke`` through the +model. Where ``UsageCallback`` records raw token counts, this one prices each +response through a :data:`~composer.llm.pricing.PriceProvider` (the model's pricing +curve, curried on model name) and adds the result to a running total. + +Implemented as an :class:`~langchain_core.callbacks.AsyncCallbackHandler` with +``run_inline = True``. The async ``on_llm_end`` is awaited on the event-loop thread +either way, but ``run_inline`` also decides *which* context it runs in: without it, +``ahandle_event`` dispatches the handler through ``asyncio.gather`` — each coroutine +wrapped in a ``Task`` against a ``copy_context()`` snapshot, so any ``ContextVar`` +the handler *sets* lands in that throwaway copy and never reaches the caller. With +``run_inline`` the handler is instead awaited directly in the caller's task and +context (``manager.ahandle_event`` line ~437), so its contextvar reads and writes +are visible to the surrounding LLM call. This matters because the accumulator +participates in contextvar state, not just its own counter. On the sync ``invoke`` +path LangChain still drives the coroutine to completion.""" + +from typing import Any + +from langchain_core.callbacks import AsyncCallbackHandler +from langchain_core.messages import AIMessage +from langchain_core.outputs import ChatGeneration, LLMResult + +from graphcore.utils import get_normalized_token_usage +from composer.llm.pricing import PriceProvider +from .budget import accumulate_cost + + +class CostAccumulator(AsyncCallbackHandler): + """Prices each LLM response and accumulates the total into :attr:`total_cost`. + + ``price_provider`` maps a call's input-token count to its per-MTok + :class:`~composer.llm.pricing.PriceTier`. ``long_cache`` selects the 1-hour + cache-write rate over the 5-minute one; the whole conversation is assumed to + share a single cache TTL (see ``builder_for``).""" + + # Route through the direct-await dispatch branch so the handler runs in the + # caller's task/context (not a gather-spawned Task with a copied context): + # required for the handler's contextvar reads/writes to reach the caller. + run_inline = True + + def __init__(self, price_provider: PriceProvider, *, long_cache: bool = False) -> None: + self._price_provider = price_provider + self._long_cache = long_cache + self.total_cost: float = 0.0 + + async def on_llm_end(self, response: LLMResult, **kwargs: Any) -> None: + try: + generation = response.generations[0][0] + except IndexError: + return + if not isinstance(generation, ChatGeneration): + return + msg = generation.message + if isinstance(msg, AIMessage): + accumulate_cost(self._cost_of(msg)) + + def _cost_of(self, msg: AIMessage) -> float: + """USD cost of a single response, in dollars. Zero for models with no + pricing-table entry.""" + usage = get_normalized_token_usage(msg) + tier = self._price_provider(usage["total_input_tokens"]) + if tier is None: + return 0.0 + + cache_read = usage["cache_read_tokens"] + cache_write = usage["cache_write_tokens"] + # Fresh input is the total minus the two cache buckets, which the tier + # prices separately. Clamp against provider rounding wobble. + fresh_input = max(0, usage["total_input_tokens"] - cache_read - cache_write) + cache_write_rate = tier.cache_write_1h if self._long_cache else tier.cache_write + + # thinking_tokens are a subset of total_output_tokens and bill at the + # output rate, so they need no separate term here. + raw_cost_pre_scale = ( + fresh_input * tier.input + + cache_read * tier.cache_read + + cache_write * cache_write_rate + + usage["total_output_tokens"] * tier.output + ) + return raw_cost_pre_scale / 1_000_000 diff --git a/composer/diagnostics/timing.py b/composer/diagnostics/timing.py index e99760ce..dd3c7704 100644 --- a/composer/diagnostics/timing.py +++ b/composer/diagnostics/timing.py @@ -219,6 +219,9 @@ def get_run_summary() -> RunSummary: """ return _run_summary.get() or RunSummary() +def get_run_summary_or_none() -> RunSummary | None: + return _run_summary.get() + def install_run_summary(summary: RunSummary) -> None: """Install ``summary`` as the active aggregator for the rest of the run.""" diff --git a/composer/foundry/artifacts.py b/composer/foundry/artifacts.py index 52bebe7d..f877b874 100644 --- a/composer/foundry/artifacts.py +++ b/composer/foundry/artifacts.py @@ -18,7 +18,7 @@ from composer.foundry.runner import infer_test_dir from composer.spec.artifacts import ArtifactStore from composer.spec.context import SourceCode -from composer.spec.cvl_generation import SkippedProperty +from composer.authoring.state import SkippedProperty from composer.spec.gen_types import ( FOUNDRY_DELIVERABLE_DIR, FOUNDRY_INTERNAL_DIR, under_project, ) diff --git a/composer/foundry/author.py b/composer/foundry/author.py index 4b367ec8..725c849a 100644 --- a/composer/foundry/author.py +++ b/composer/foundry/author.py @@ -8,7 +8,7 @@ Workflow shape: -* Single ``curr_test: str`` buffer per batch (one ``.t.sol`` file), written +* Single ``curr_spec`` buffer per batch (one ``.t.sol`` file), written via ``put_test_raw``. No put-time compile check; ``forge_test`` is the gate. * A feedback judge (``feedback_tool``) reviews the draft against the batch's properties. Publish requires both a green unseeded ``forge_test`` run AND a @@ -21,51 +21,63 @@ * No prover-config editor — foundry projects are assumed pre-configured. """ +from contextlib import asynccontextmanager from dataclasses import dataclass +from pathlib import Path import asyncio from typing import ( - Awaitable, Callable, Literal, NotRequired, Protocol, override + AsyncIterator, Awaitable, Callable, Literal, NotRequired, Protocol, + Sequence, override, overload ) from typing_extensions import TypedDict +from langchain_core.tools import BaseTool from langgraph.graph import MessagesState from langgraph.types import Command from pydantic import BaseModel, Field -from graphcore.graph import FlowInput, tool_state_update +from graphcore.graph import CacheMarker, FlowInput, RawPromptInput, tool_state_update from graphcore.summary import SummaryConfig from graphcore.tools.schemas import ( WithAsyncDependencies, WithAsyncImplementation, WithImplementation, WithInjectedId, WithInjectedState, ) -from composer.pipeline.core import GaveUp +from composer.pipeline.plugin_api import ProvidedTools +from composer.foundry.plugin import FoundryState, FoundryTools + +from composer.authoring.buffer import ( + SpecBuffer, SpecBufferSet, apply_spec_update, get_spec_tool, +) +from composer.authoring.judge import ( + FeedbackThunk, JudgeBuilder, JudgeState, RebuttalBase, build_feedback_judge, +) +from composer.authoring.state import SkippedProperty +from composer.authoring.tools import give_up_tool, skip_tools +from composer.pipeline.core import Curtailed, GaveUp, ToolBinder, ToolExtension from composer.spec.context import FoundryGeneration, FoundryJudge, WorkflowContext -from composer.spec.cvl_generation import ( - PropertyFeedbackProtocol, RebuttalBase, SkippedProperty, +from composer.diagnostics.budget import ( + BudgetExceeded, budget_monitor, budget_pressure, + raise_budget_exceeded, constraint_sort_to_noun, ) -from composer.spec.feedback import PropertyFeedback from composer.spec.gen_types import TypedTemplate -from composer.spec.graph_builder import bind_standard, run_to_completion -from composer.spec.types import PropertyFormulation +from composer.spec.graph_builder import run_to_completion +from composer.spec.types import CheckName, PropertyFormulation, PropertyTitle from composer.spec.system_model import ContractComponentInstance, component_context from composer.spec.service_host import ServiceHost -from composer.spec.util import uniq_thread_id -from composer.tools.thinking import RoughDraftState, get_rough_draft_tools from composer.ui.tool_display import ( - suppress_ack, tool_display, + ToolDisplay, suppress_ack, tool_display, ) -from composer.foundry.runner import get_forge_test_tool +from composer.foundry.runner import get_forge_test_tool, infer_test_dir +from composer.authoring.state import make_validation_stamper from composer.foundry.state import ( FEEDBACK, FORGE_TEST_VALIDATION_KEY, FOUNDRY_JUDGE_KEY, - FoundryTestExtra, FoundryGenerationInput, FoundryGenerationState, PropertyTestMapping, check_foundry_completion, - make_foundry_validation_stamper, validate_property_tests, ) @@ -84,10 +96,10 @@ class GeneratedFoundryTest(BaseModel): # gating unseeded run, and the author's expected-failure markings (test # name -> reason). Together they give every test a pass / expected-failure # status without trusting the model's own transcription. - expected_failures: dict[str, str] = Field(default_factory=dict) - ran_tests: list[str] = Field(default_factory=list) + expected_failures: dict[CheckName, str] = Field(default_factory=dict) + ran_tests: list[CheckName] = Field(default_factory=list) - def property_units(self) -> list[tuple[str, list[str]]]: + def property_checks(self) -> list[tuple[PropertyTitle, list[CheckName]]]: """Property title -> the foundry test names that demonstrate it (the report's `ReportableResult` adapter; pairs with the structurally-shared ``skipped`` field).""" return [(m.property_title, m.tests) for m in self.property_tests] @@ -101,7 +113,7 @@ def output_link(self) -> str | None: return None # foundry has no external run service -type BatchFoundryResult = GeneratedFoundryTest | GaveUp +type BatchFoundryResult = GeneratedFoundryTest | Curtailed[GeneratedFoundryTest] | GaveUp # --------------------------------------------------------------------------- @@ -113,11 +125,11 @@ def output_link(self) -> str | None: label=lambda p: f"Putting test draft ({len(p.get('test_source', ''))} chars)", result=suppress_ack("Put test result", ("Accepted",)), ) -class PutTestRaw(WithImplementation[Command], WithInjectedId): +class PutTestRaw(WithImplementation[Command | str], WithInjectedId): """ Put a foundry test file into the working buffer. - The provided source replaces the entire ``curr_test`` buffer. There is no + The provided source replaces the entire test buffer. There is no put-time compile check — call ``forge_test`` to verify the draft actually builds and passes. ``forge_test``'s green stamp is invalidated by any subsequent ``put_test_raw``, so call ``forge_test`` *after* you're done @@ -133,106 +145,43 @@ class PutTestRaw(WithImplementation[Command], WithInjectedId): ) @override - def run(self) -> Command: - return tool_state_update( - tool_call_id=self.tool_call_id, - content="Accepted", - curr_test=self.test_source, - ) + def run(self) -> Command | str: + return apply_spec_update(tool_call_id=self.tool_call_id, text=self.test_source) -@tool_display("Reading current test draft", None) -class GetTestTool(WithInjectedState[FoundryTestExtra], WithImplementation): - """ - Retrieve the textual representation of the current foundry test. - """ - def run(self) -> str: - if self.state["curr_test"] is None: - return "No test draft written" - return self.state["curr_test"] - -@tool_display( - lambda p: f"Skipping property `{p.get('property_title', '?')}`", - suppress_ack("Skip result", ("Recorded skip",)), -) -class _RecordSkipSchema( - WithInjectedId, - # deps: the batch's property titles - WithAsyncDependencies[Command, list[str]], -): - """ +_SKIP_DESCRIPTION = """ Declare that you are skipping a property from the batch. You must provide the property's title and a justification. Skipping excludes the property from the publish-time property→test mapping check; only use after a genuine attempt to formalize. """ - property_title: str = Field( - description="The snake_case title of the property from the batch listing" - ) - reason: str = Field( - description="Justification for why this property cannot be formalized as a foundry test" - ) - - @override - async def run(self) -> Command: - with self.tool_deps() as titles: - if self.property_title not in titles: - return tool_state_update( - self.tool_call_id, - f"Unknown property title {self.property_title!r}. Must be one " - f"of: {', '.join(titles)}.", - ) - if not self.reason.strip(): - return tool_state_update( - self.tool_call_id, - "A non-empty justification is required when skipping a property.", - ) - skip = SkippedProperty( - property_title=self.property_title, - reason=self.reason, - ) - return tool_state_update( - self.tool_call_id, - f"Recorded skip for property {self.property_title}.", - skipped=[skip], - ) +_SKIP_REASON = "Justification for why this property cannot be formalized as a foundry test" -@tool_display( - lambda p: f"Un-skipping property `{p.get('property_title', '?')}`", - suppress_ack("Unskip result", ("Removed skip",)), -) -class _UnskipSchema( - WithInjectedId, - # deps: the batch's property titles - WithAsyncDependencies[Command, list[str]], -): - """ - Remove a previously declared skip for a property. Use this if you later - find a way to formalize a property you previously skipped. +_GET_TEST_DESCRIPTION = """ + Retrieve the textual representation of the current foundry test. """ - property_title: str = Field( - description="The snake_case title of the property to un-skip" - ) - @override - async def run(self) -> Command: - with self.tool_deps() as titles: - if self.property_title not in titles: - return tool_state_update( - self.tool_call_id, - f"Unknown property title {self.property_title!r}. Must be one " - f"of: {', '.join(titles)}.", - ) - # Sentinel reason "" — _merge_skips drops empty-reason entries. - skip = SkippedProperty(property_title=self.property_title, reason="") - return tool_state_update( - self.tool_call_id, - f"Removed skip for property {self.property_title}.", - skipped=[skip], - ) +@overload +def get_test_tool[S: SpecBufferSet](ty: type[S]) -> BaseTool: ... + + +@overload +def get_test_tool[S: SpecBuffer](ty: type[S]) -> BaseTool: ... + + +def get_test_tool(ty: type) -> BaseTool: + """The read-back tool over the test buffer, named ``get_test`` for both the author and its + judge.""" + return get_spec_tool( + ty, + name="get_test", + description=_GET_TEST_DESCRIPTION, + missing="No test draft written", + display=ToolDisplay("Reading current test draft", None), + ) @tool_display(lambda p: f"Expecting test `{p['test_name']}` to fail", None) class ExpectTestFailure(WithAsyncImplementation[Command], WithInjectedId): @@ -321,19 +270,9 @@ class Rebuttal(RebuttalBase): ) -type _FeedbackImplThunk = Callable[ - [str, list[SkippedProperty], list[Rebuttal], str], - Awaitable[PropertyFeedbackProtocol], -] -"""``(test_source, skipped, rebuttals, within_tool) -> PropertyFeedback``. -``within_tool`` is the calling ``FeedbackTool``'s ``tool_call_id``, plumbed -through to the judge's ``run_to_completion`` so its UI panel anchors under -the parent tool widget.""" - - @dataclass class FeedbackDependencies: - thunk: _FeedbackImplThunk + thunk: FeedbackThunk[Rebuttal] stamper: Callable[[FoundryGenerationState], dict[str, str]] @@ -369,11 +308,20 @@ class FeedbackTool( @override async def run(self) -> Command | str: - if self.state["curr_test"] is None: + if self.state["curr_spec"] is None: return "No test written" + if budget_pressure(): + # Don't launch a judge that would be terminated on its first + # monitor tick; the author's budget warning already tells it + # feedback approval is no longer required. + return ( + "Good? False\nFeedback:\nThe feedback judge was not run due to " + "budget constraints. See the system alert: feedback approval is " + "no longer required for this task." + ) with self.tool_deps() as deps: res = await deps.thunk( - self.state["curr_test"], + self.state["curr_spec"], self.state["skipped"], self.rebuttals, self.tool_call_id, @@ -388,62 +336,59 @@ async def run(self) -> Command | str: return result +@component_context +class _FoundryJudgeParams(TypedDict): + """Render variables for ``foundry_feedback_prompt.j2``. + + ``sort`` is what the shared application-context partial gates its "pre-existing codebase being + extended" wording on. Foundry only ever verifies an existing project, so it is fixed here — + naming it also keeps the template renderable under ``COMPOSER_STRICT_TEMPLATES``, which is what + the fuzzer runs it as now that the template is declared.""" + properties: list[PropertyFormulation] + context: ContractComponentInstance | None + sort: Literal["existing"] + + +_FoundryJudgeTemplate = TypedTemplate[_FoundryJudgeParams]("foundry_feedback_prompt.j2") +class _NoParams(TypedDict): + pass + + +_FoundryJudgeSystemTemplate = TypedTemplate[_NoParams]("foundry_property_judge_system_prompt.j2") + + def _build_feedback_thunk( judge_ctx: WorkflowContext[FoundryJudge], env: ServiceHost, props: list[PropertyFormulation], component: ContractComponentInstance | None, -) -> _FeedbackImplThunk: - """Compile the feedback-judge graph and wrap it in the thunk - ``FeedbackTool`` invokes. The judge follows the CVL property judge's - review protocol (rough draft + persistent memory + read-back of the - artifact under review) with the foundry tool surface: project source - tools + the cheatcode RAG.""" - - class JudgeExtra(RoughDraftState): - curr_test: str - - class ST(MessagesState, JudgeExtra): - result: NotRequired[PropertyFeedback] - - class TestJudgeInput(FlowInput, JudgeExtra): - pass - - def did_rough_draft_read(s: ST, _) -> str | None: - if not s["did_read"]: - return "Completion REJECTED: never read rough draft for review" - return None - - workflow = bind_standard( - env.builder_heavy().with_tools(env.source_tools).with_tools(env.rag_tools), - ST, - validator=did_rough_draft_read, - ).with_input( - TestJudgeInput - ).with_initial_prompt_template( - "foundry_feedback_prompt.j2", properties=props, context=component, - ).with_sys_prompt_template( - "foundry_property_judge_system_prompt.j2" - ).with_tools( - [*get_rough_draft_tools(ST), judge_ctx.get_memory_tool(), GetTestTool.as_tool("get_test")] - ).compile_async() - - async def thunk( - test_source: str, - skipped: list[SkippedProperty], - rebuttals: list[Rebuttal], - within_tool: str, - ) -> PropertyFeedbackProtocol: - input_parts: list[str | dict] = [ +) -> FeedbackThunk[Rebuttal]: + """The foundry feedback judge. The shared judge supplies the review protocol (rough draft + + persistent memory + enforced read-back of the file under review); what is foundry's own is the + prompt pair and the fact that the skips and rebuttals are stated as input text rather than + rendered into the prompt template.""" + + def apply_prompt( + builder: JudgeBuilder, _spec: str, + _skipped: Sequence[SkippedProperty], _rebuttals: Sequence[Rebuttal], + ) -> JudgeBuilder: + return _FoundryJudgeTemplate.bind({ + "properties": props, "context": component, "sort": "existing", + }).render_to(builder.with_initial_prompt_template) + + def input_parts( + test_source: str, skipped: Sequence[SkippedProperty], rebuttals: Sequence[Rebuttal] + ) -> list[str | dict]: + parts: list[str | dict] = [ "The proposed foundry test file is", test_source, ] if skipped: - input_parts.append("The following properties were explicitly skipped by the author:") + parts.append("The following properties were explicitly skipped by the author:") for s in skipped: - input_parts.append(f" Property {s.property_title}: {s.reason}") + parts.append(f" Property {s.property_title}: {s.reason}") if rebuttals: - input_parts.append( + parts.append( "The author has filed the following rebuttals against feedback " "from prior rounds. Evaluate each per the rebuttal rules in your " "instructions. Empirical evidence types (`compilation_failure`, " @@ -452,26 +397,25 @@ async def thunk( "not a veto." ) for i, r in enumerate(rebuttals, 1): - input_parts.append( + parts.append( f" Rebuttal {i} [{r.evidence_type}]\n" f" Addressing: {r.prior_feedback_reference}\n" f" Evidence: {r.evidence}" ) - res = await run_to_completion( - workflow, - TestJudgeInput( - input=input_parts, curr_test=test_source, - memory=None, did_read=False, - ), - thread_id=uniq_thread_id("foundry-feedback"), - recursion_limit=judge_ctx.recursion_limit, - description="Foundry test feedback judge", - within_tool=within_tool, - ) - assert "result" in res - return res["result"] + return parts - return thunk + return build_feedback_judge( + ctx=judge_ctx, + env=env, + apply_system=lambda b: _FoundryJudgeSystemTemplate.bind({}).render_to( + b.with_sys_prompt_template + ), + apply_prompt=apply_prompt, + input_parts=input_parts, + readback=get_test_tool(JudgeState), + description="Foundry test feedback judge", + thread_prefix="foundry-feedback", + ) # --------------------------------------------------------------------------- @@ -484,7 +428,7 @@ class PublishResultTool( WithInjectedState[FoundryGenerationState], WithInjectedId, # deps: the batch's property titles - WithAsyncDependencies[Command | str, list[str]], + WithAsyncDependencies[Command | str, list[PropertyTitle]], ): """ Call to signal completion. The publish is gated on the required @@ -513,17 +457,22 @@ class PublishResultTool( async def run(self) -> Command | str: if (err := check_foundry_completion(self.state)) is not None: return err - ran = self.state["last_test_names"] - if ran is None: - # Unreachable in practice — the forge_test stamp required above - # implies a run recorded its test names — but defend anyway. - return "Completion REJECTED: no forge_test run has been recorded." - with self.tool_deps() as titles: - err = validate_property_tests( - self.property_tests, self.state["skipped"], titles, ran, - ) - if err is not None: - return err + if not budget_pressure(): + # The forge-ground-truth cross-check requires a recorded run; in + # the budget wrap-up window (where the agent is told to delete + # failing tests and publish without re-running forge) the declared + # mapping is accepted as-is. + ran = self.state["last_test_names"] + if ran is None: + # Unreachable in practice — the forge_test stamp required above + # implies a run recorded its test names — but defend anyway. + return "Completion REJECTED: no forge_test run has been recorded." + with self.tool_deps() as titles: + err = validate_property_tests( + self.property_tests, self.state["skipped"], titles, ran, + ) + if err is not None: + return err return tool_state_update( self.tool_call_id, "Accepted", @@ -533,25 +482,10 @@ async def run(self) -> Command | str: ) -@tool_display( - label=lambda p: f"Giving up on foundry-test generation: {p['reason']}", - result=None, -) -class GiveUpTool(WithImplementation[Command], WithInjectedId): - """ +_GIVE_UP_DESCRIPTION = """ Last-resort exit when you've exhausted other mechanisms to complete the task. The batch will be reported as failed with your ``reason``. """ - reason: str = Field(description="Why you are giving up on this batch") - - @override - def run(self) -> Command: - return tool_state_update( - self.tool_call_id, - "Accepted", - failed=True, - result=self.reason, - ) # --------------------------------------------------------------------------- @@ -562,7 +496,7 @@ def run(self) -> Command: class FoundryGenerationSummaryConfig(SummaryConfig[FoundryGenerationState]): """Summarization prompts for the foundry author when the context window fills up. Same role as ``PropertyGenerationConfig`` in the CVL author, - reworded for the foundry workflow (``curr_test`` not ``curr_spec``).""" + reworded for the foundry workflow (a test file, not a CVL spec).""" @override def get_summarization_prompt(self, state: FoundryGenerationState) -> str: @@ -610,6 +544,13 @@ def get_resume_prompt(self, state: FoundryGenerationState, summary: str) -> str: # Top-level batch entry # --------------------------------------------------------------------------- +#: The foundry tool extension: contributions come from plugins deriving +#: ``FoundryTools``, dispatched via their ``foundry_tools`` hook. +_FOUNDRY_TOOLS = ToolExtension( + provider=FoundryTools, project=lambda p: p.foundry_tools +) + + @component_context class FoundryPropertyGenParams(TypedDict): """Per-batch render variables for ``foundry_property_generation_prompt.j2``. @@ -625,6 +566,21 @@ class FoundryPropertyGenParams(TypedDict): "foundry_property_generation_prompt.j2" ) +_BUDGET_WRAPUP_MESSAGE = """ + +You have almost exceeded the {resource} budget for this task. Wrap up IMMEDIATELY; +a partial test file is better than going over budget. Concretely: + +- The forge-test and feedback validation requirements on publishing have been lifted. You + no longer need approval from the feedback judge — ignore any pending or future feedback, + including a judge response saying it was terminated. +- Do NOT start new forge runs or research. +- Delete any tests that do not currently compile or pass. +- Skip (`record_skip`) every property you have not gotten to work, citing budget exhaustion. +- Then publish what remains via the `result` tool. + +""" + async def batch_foundry_test_generation( ctx: WorkflowContext[FoundryGeneration], @@ -637,13 +593,14 @@ async def batch_foundry_test_generation( description: str, forge_binary: str = "forge", forge_timeout_s: int = 600, - forge_sem : asyncio.Semaphore + forge_sem : asyncio.Semaphore, + tool_provider: ToolBinder[ContractComponentInstance], ) -> BatchFoundryResult: """Author one batch of foundry tests covering ``props``. The graph terminates when the agent calls ``result`` (publish) or ``give_up``. Both ``forge_test`` and the feedback judge must have stamped - the *current* ``curr_test`` for ``result`` to be accepted. + the *current* buffer for ``result`` to be accepted. Caller responsibilities: @@ -656,6 +613,9 @@ async def batch_foundry_test_generation( ``composer.foundry.env.build_foundry_env``. * ``contract_name`` / ``component`` / ``props`` are bound into the initial prompt (``foundry_property_generation_prompt.j2``). + * ``tool_provider`` is the driver's tool binder for this batch, + dispatched here with the foundry extension and a reader that stages + :class:`FoundryState` from the author's live graph state. ``ctx`` is marked ``FoundryGeneration`` so its cache namespace stays distinct from a co-located CVL run's. @@ -671,11 +631,42 @@ async def batch_foundry_test_generation( "sort": "existing" }) + # Stage the foundry analog of the prover's ProverState for contributed tools: + # the (in-situ) project root, the configured test dir, and the live draft + # buffer. Nothing is materialized — foundry runs against the project as-is — + # so the context manager is trivial; the shape leaves the backend free to + # stage a copy later without touching the plugin contract. + root = Path(project_root).resolve() + test_dir = root / infer_test_dir(root) + + @asynccontextmanager + async def _staged_state(st: FoundryGenerationState) -> AsyncIterator[FoundryState]: + yield FoundryState( + working_dir=root, + test_dir=test_dir, + curr_test=st["curr_spec"], + ) + + tools = await tool_provider( + _FOUNDRY_TOOLS, FoundryGenerationState, _staged_state + ) + + sys_prompt: list[RawPromptInput | type[CacheMarker]] = [ + lambda load: load("foundry_property_generation_system_prompt.j2"), + ] + added_tools: list[BaseTool] = [] + for inj in tools: + added_tools.extend(inj.tools) + if isinstance(inj.system_prompt_injection, list): + sys_prompt.extend(inj.system_prompt_injection) + else: + sys_prompt.append(inj.system_prompt_injection) + titles = [p.title for p in props] judge_ctx = ctx.child(FOUNDRY_JUDGE_KEY) feedback_deps = FeedbackDependencies( thunk=_build_feedback_thunk(judge_ctx, env, props, component), - stamper=make_foundry_validation_stamper(FEEDBACK), + stamper=make_validation_stamper(FEEDBACK), ) builder = ( @@ -687,25 +678,38 @@ async def batch_foundry_test_generation( .with_tools(env.rag_tools) .with_tools([ PutTestRaw.as_tool("put_test_raw"), - GetTestTool.as_tool("get_test"), - _RecordSkipSchema.bind(titles).as_tool("record_skip"), - _UnskipSchema.bind(titles).as_tool("unskip_property"), + get_test_tool(FoundryGenerationState), + *skip_tools( + titles, + skip_description=_SKIP_DESCRIPTION, + skip_reason=_SKIP_REASON, + ), ExpectTestFailure.as_tool("expect_test_failure"), ExpectTestPassage.as_tool("expect_test_passage"), forge_test_tool, FeedbackTool.bind(feedback_deps).as_tool("feedback_tool"), PublishResultTool.bind(titles).as_tool("result"), - GiveUpTool.as_tool("give_up"), + give_up_tool( + name="give_up", description=_GIVE_UP_DESCRIPTION, + label="foundry-test generation", + reason_description="Why you are giving up on this batch", + ), ctx.get_memory_tool(), ]) - .with_sys_prompt_template("foundry_property_generation_system_prompt.j2") + .with_tools(added_tools) + .with_sys_prompt(sys_prompt) .inject(lambda b: bound_template.render_to(b.with_initial_prompt_template)) .with_summary_config(FoundryGenerationSummaryConfig()) + .with_monitor(budget_monitor( + warning_message=lambda _s, c: _BUDGET_WRAPUP_MESSAGE.format(resource=constraint_sort_to_noun(c)), + state_transformer=lambda _s, _c: {"required_validations": [], "budget_curtailed": True}, + on_overbudget=raise_budget_exceeded, + )) ) graph = builder.compile_async() init_state = FoundryGenerationInput( - curr_test=None, + curr_spec=None, input=[], required_validations=[FORGE_TEST_VALIDATION_KEY, FEEDBACK], skipped=[], @@ -714,24 +718,32 @@ async def batch_foundry_test_generation( expected_failures={}, last_test_names=None, failed=None, + budget_curtailed=False, ) tid, mnem = await ctx.thread_and_mnemonic() - res_state = await run_to_completion( - graph, - init_state, - thread_id=tid, - description=f"{description} ({mnem})", - recursion_limit=ctx.recursion_limit, - ) + try: + res_state = await run_to_completion( + graph, + init_state, + thread_id=tid, + description=f"{description} ({mnem})", + recursion_limit=ctx.recursion_limit, + ) + except BudgetExceeded as e: + return Curtailed(None, detail=str(e)) assert "result" in res_state assert res_state["failed"] is not None if res_state["failed"]: + if res_state["budget_curtailed"]: + # A give-up issued after the wrap-up order isn't a considered "this batch is + # unformalizable" judgment — it's the budget talking. Keep the agent's account. + return Curtailed(None, detail=res_state["result"]) return GaveUp(reason=res_state["result"]) - draft = res_state["curr_test"] + draft = res_state["curr_spec"] assert draft is not None - return GeneratedFoundryTest( + generated = GeneratedFoundryTest( commentary=res_state["result"], test_source=draft, skipped=res_state["skipped"], @@ -739,3 +751,7 @@ async def batch_foundry_test_generation( expected_failures=res_state["expected_failures"], ran_tests=res_state["last_test_names"] or [], ) + if res_state["budget_curtailed"]: + # Published under lifted gates: hand it back as an explicitly unreliable partial. + return Curtailed(generated) + return generated diff --git a/composer/foundry/entry.py b/composer/foundry/entry.py index 85fa164c..74d50711 100644 --- a/composer/foundry/entry.py +++ b/composer/foundry/entry.py @@ -23,7 +23,7 @@ from composer.core.user import get_uid from composer.diagnostics.timing import RunSummary -from composer.input.parsing import Arg, add_protocol_args +from composer.input.parsing import Arg, add_extra_context_args, add_protocol_args from composer.input.types import DEFAULT_RECURSION_LIMIT, RAGDBOptions, ExtendedModelOptions from composer.io.multi_job import HandlerFactory from composer.io.thread_logging import RunDataLogger @@ -37,6 +37,7 @@ FoundryPhase, FoundryPipelineResult, backend ) from composer.pipeline.cli import cli_pipeline, user_ns, AtExit +from composer.pipeline.ptypes import DEFAULT_MAX_CPU_TASKS from composer.pipeline.ecosystem import EVM _log = logging.getLogger(__name__) @@ -63,6 +64,7 @@ class FoundryArgs(ExtendedModelOptions, FoundryRAGDBOptions, Protocol): main_contract: str system_doc: str | None max_concurrent: int + max_cpu_tasks: int cache_ns: str | None memory_ns: str | None interactive: bool @@ -71,6 +73,9 @@ class FoundryArgs(ExtendedModelOptions, FoundryRAGDBOptions, Protocol): forge_binary: str forge_timeout_s: int max_forge_runners: int + budget: str | None + time_budget: float | None + extra_context: list[str] | None @property def threat_model(self) -> None: @@ -124,6 +129,7 @@ def _build_parser() -> argparse.ArgumentParser: parser.add_argument("main_contract", help="Main contract as path:ContractName") parser.add_argument("system_doc", nargs="?", default=None, help="Path to the design document (text or PDF). Optional — auto-discovered from the project when omitted.") parser.add_argument("--max-concurrent", type=int, default=4, help="Max concurrent agents (default: 4)") + parser.add_argument("--max-cpu-tasks", type=int, default=DEFAULT_MAX_CPU_TASKS, help=f"Max concurrent CPU-bound tasks — toolchain builds and the like (default: {DEFAULT_MAX_CPU_TASKS})") parser.add_argument("--max-forge-runners", default=1, type=int, help="Max concurrent forge runners (default: 1)") parser.add_argument("--cache-ns", default=None, help="Cache namespace (enables cross-run caching)") parser.add_argument("--memory-ns", default=None, help="Memory namespace (default: thread id)") @@ -131,6 +137,9 @@ def _build_parser() -> argparse.ArgumentParser: parser.add_argument("--max-bug-rounds", type=int, default=3, help="Max bug-extraction rounds per component (default: 3)") parser.add_argument("--forge-binary", default="forge", help="`forge` executable on PATH (default: forge)") parser.add_argument("--forge-timeout-s", type=int, default=600, help="Per-`forge test` invocation timeout in seconds (default: 600)") + parser.add_argument("--budget", default=None, help="Path to a run-budget file (JSON or YAML): {total: USD, caps: {phase: USD, ...}}. Omit to run unbudgeted.") + parser.add_argument("--time-budget", default=None, type=float, help="Total wall time to run the entire execution. Omit to run without in process limit") + add_extra_context_args(parser) parser.set_defaults(threat_model=None) return parser diff --git a/composer/foundry/env.py b/composer/foundry/env.py index 21974d38..20721187 100644 --- a/composer/foundry/env.py +++ b/composer/foundry/env.py @@ -25,6 +25,7 @@ from composer.rag.db import ComposerRAGDB +from composer.pipeline.ecosystem import EVM from composer.spec.source.source_env import ( build_basic_source_tools, build_source_tools, ) @@ -64,6 +65,7 @@ def build_foundry_env( store, source_question_ns, recursion_limit=recursion_limit, + ecosystem=EVM, ) rag = tuple(foundry_cheatcode_tools(rag_db)) diff --git a/composer/foundry/pipeline.py b/composer/foundry/pipeline.py index 7c5ba396..2f866dba 100644 --- a/composer/foundry/pipeline.py +++ b/composer/foundry/pipeline.py @@ -22,7 +22,7 @@ import enum import logging from dataclasses import dataclass -from typing import Awaitable, Callable, override +from typing import Awaitable, Callable, override, Sequence, Any from composer.foundry.author import ( GeneratedFoundryTest, batch_foundry_test_generation, @@ -32,12 +32,15 @@ from composer.pipeline.core import ( Formalizer, PreparedSystem, PipelineRun, GaveUp, SystemAnalysisSpec, - CorePhases, CorePipelineResult, - COMMON_SYSTEM_CACHE_KEY + CorePhases, CorePipelineResult, ToolBinder, ) +from composer.foundry.plugin import FoundryTools +from composer.pipeline.ptypes import Curtailed from composer.pipeline.ecosystem import main_instance +from composer.pipeline.keys import COMMON_SYSTEM_CACHE_KEY from composer.foundry.artifacts import FoundryTestArtifact -from composer.spec.source.report.collect import ReportComponentInput, Verdict +from composer.spec.source.report.collect import Formalized, Verdict +from composer.spec.source.report.schema import RuleName from composer.spec.context import ( WorkflowContext, SourceCode, FoundryGeneration ) @@ -108,6 +111,8 @@ class FoundryPhase(enum.Enum): REPORT = "report" class FoundryFormalizer(Formalizer[GeneratedFoundryTest, ContractComponentInstance]): + tool_provider_type = FoundryTools + def __init__(self, conf: _ForgeRunConfig): super().__init__(GeneratedFoundryTest, "foundry") self.conf = conf @@ -119,8 +124,9 @@ async def formalize( feat: ContractComponentInstance, props: list[PropertyFormulation], ctx: WorkflowContext[GeneratedFoundryTest], - run: PipelineRun - ) -> GeneratedFoundryTest | GaveUp: + run: PipelineRun, + extra_tools: ToolBinder[ContractComponentInstance] + ) -> GeneratedFoundryTest | Curtailed[GeneratedFoundryTest] | GaveUp: return await batch_foundry_test_generation( ctx=ctx.abstract(FoundryGeneration), project_root=run.source.project_root, @@ -131,12 +137,15 @@ async def formalize( forge_binary=self.conf.forge_binary, forge_sem=self.conf.forge_sem, forge_timeout_s=self.conf.forge_timeout_s, - props=props + props=props, + tool_provider=extra_tools, ) - + @override - async def fetch_verdicts(self, inp: ReportComponentInput[GeneratedFoundryTest]) -> dict[str, Verdict]: - return await _foundry_verdicts(inp) + async def fetch_verdicts( + self, formalized: Formalized[GeneratedFoundryTest] + ) -> dict[RuleName, Verdict]: + return await _foundry_verdicts(formalized) @dataclass class FoundrySystem(PreparedSystem[GeneratedFoundryTest, ContractComponentInstance, ContractInstance]): @@ -163,10 +172,17 @@ class FoundryBackend: foundry_conf: _ForgeRunConfig + async def preflight(self, run: PipelineRun[FoundryPhase, None]) -> None: + """Nothing to do ahead of analysis. Foundry authors `.t.sol` into a project `forge` already + builds, so there is no workspace to prepare; the existing project is the precondition (a + `forge build` smoke test would be the natural thing to add here).""" + return None + async def prepare_system( self, analyzed: SourceApplication, - run: PipelineRun[FoundryPhase, None] + run: PipelineRun[FoundryPhase, None], + preflight: None, ) -> PreparedSystem[GeneratedFoundryTest, ContractComponentInstance, ContractInstance]: return FoundrySystem( main_instance( diff --git a/composer/foundry/plugin.py b/composer/foundry/plugin.py new file mode 100644 index 00000000..9351d0e2 --- /dev/null +++ b/composer/foundry/plugin.py @@ -0,0 +1,49 @@ +"""The foundry backend's plugin-extension surface. + +What a plugin imports to contribute tools to the foundry test author: the +:class:`FoundryTools` provider class (deriving it IS the declaration — see +``composer.pipeline.plugin_api``) and the :class:`FoundryState` view its hooks can +stage at tool-invocation time. The prover counterpart is +``composer.spec.source.plugin``. +""" + +import pathlib +from abc import abstractmethod +from dataclasses import dataclass +from typing import AsyncContextManager, Callable, Sequence + +from composer.pipeline.plugin_api import ( + FormalizationTool, PipelinePlugin, PluginToolContext, ProvidedTools, +) +from composer.spec.system_model import FeatureUnit +from composer.spec.types import PropertyFormulation + + +@dataclass +class FoundryState: + # The foundry project root the run executes in. + working_dir: pathlib.Path + # The project's configured test directory (absolute; from foundry.toml's + # default profile). May not exist until a draft is first staged into it. + test_dir: pathlib.Path + # The author's current .t.sol draft buffer; None until a draft is written. + curr_test: str | None + +type FoundryStateReader[T] = Callable[[T], AsyncContextManager[FoundryState]] + + +class FoundryTools[U: FeatureUnit](PipelinePlugin[U]): + """Contributes tools to the foundry test author; the staged view is + :class:`FoundryState`. See ``composer.spec.source.plugin.CertoraProverTools`` + for the reader contract.""" + + @abstractmethod + async def foundry_tools[T]( + self, + comp: U, + prop: Sequence[PropertyFormulation], + tool_context: PluginToolContext[FormalizationTool], + st: type[T], + state_reader: FoundryStateReader[T] + ) -> ProvidedTools | None: + ... diff --git a/composer/foundry/report.py b/composer/foundry/report.py index ed24231f..882337a7 100644 --- a/composer/foundry/report.py +++ b/composer/foundry/report.py @@ -11,25 +11,25 @@ from composer.foundry.author import GeneratedFoundryTest from composer.spec.source.report.build import build_report -from composer.spec.source.report.collect import ReportComponentInput, Verdict +from composer.spec.source.report.collect import ( + Formalized, ReportComponentInput, Verdict, +) from composer.spec.source.report.schema import AutoProverReport, Outcome, RuleName _log = logging.getLogger(__name__) async def _foundry_verdicts( - inp: ReportComponentInput[GeneratedFoundryTest], + formalized: Formalized[GeneratedFoundryTest], ) -> dict[RuleName, Verdict]: """Per-test verdicts from forge ground truth: a ran test is GOOD unless the author marked it an - expected failure (BAD). No external service — read straight off the result.""" - fm = inp.formalized - if fm is None: - return {} - res = fm.result + expected failure (BAD). No external service — read straight off the result. Only ever invoked + for delivered results (collect skips gave-up / curtailed inputs).""" + res = formalized.result return { name: Verdict( outcome=Outcome.BAD if res.expected_failures.get(name) else Outcome.GOOD, - unit_file=fm.unit_file, + unit_file=formalized.unit_file, ) for name in res.ran_tests } diff --git a/composer/foundry/runner.py b/composer/foundry/runner.py index 9111d1ca..1f80f70e 100644 --- a/composer/foundry/runner.py +++ b/composer/foundry/runner.py @@ -3,7 +3,7 @@ Exposes ``ForgeTestTool`` (and a convenience ``get_forge_test_tool`` factory) that: -* Reads ``curr_test`` from injected state. +* Reads the ``curr_spec`` test buffer from injected state. * Stages it into ``/test/_composer_draft_.t.sol`` for the duration of one ``forge test`` run, then deletes the staged file. ```` is either the agent-supplied ``seed`` arg (stable across @@ -48,10 +48,11 @@ from composer.ui.tool_display import tool_display +from composer.authoring.state import make_validation_stamper +from composer.spec.types import CheckName from composer.foundry.state import ( FORGE_TEST_VALIDATION_KEY, FoundryGenerationState, - make_foundry_validation_stamper, ) @@ -81,7 +82,7 @@ class ForgeTestRunEvent(TypedDict): @dataclass(frozen=True) class _TestResult: - name: str + name: CheckName status: str reason: str | None @@ -100,7 +101,7 @@ class ForgeTestTool( """ Run the project's foundry test suite against your current draft. - The current ``curr_test`` buffer is written into + The current test buffer is written into ``/test/`` as a ``.t.sol`` file, then ``forge test --json --match-path test/`` runs. @@ -140,7 +141,7 @@ class ForgeTestTool( @override async def run(self) -> Command | str: - if self.state["curr_test"] is None: + if self.state["curr_spec"] is None: return "No test written yet. Call put_test_raw before forge_test." with self.tool_deps() as deps: @@ -160,7 +161,7 @@ async def run(self) -> Command | str: seeded = self.seed is not None staged_name = f"_composer_draft_{path_key}.t.sol" staged = test_dir / staged_name - staged.write_text(self.state["curr_test"]) + staged.write_text(self.state["curr_spec"]) try: proc = await asyncio.create_subprocess_exec( @@ -250,7 +251,7 @@ async def run(self) -> Command | str: test_names = [r.name for r in results] if clean and not seeded: - stamper = make_foundry_validation_stamper(FORGE_TEST_VALIDATION_KEY) + stamper = make_validation_stamper(FORGE_TEST_VALIDATION_KEY) return tool_state_update( tool_call_id=self.tool_call_id, content=( @@ -387,7 +388,7 @@ def _parse_forge_json(stdout: str) -> list[_TestResult] | None: report = _FORGE_REPORT.validate_python(doc) return [ _TestResult( - name=signature.split("(", 1)[0].strip(), + name=CheckName(signature.split("(", 1)[0].strip()), status=entry.status, reason=entry.reason, ) @@ -396,7 +397,7 @@ def _parse_forge_json(stdout: str) -> list[_TestResult] | None: ] -def _format_summary(results: list[_TestResult], expected_failures: dict[str, str]) -> str: +def _format_summary(results: list[_TestResult], expected_failures: dict[CheckName, str]) -> str: """Render a compact human-readable summary of the JSON results.""" if not results: return "(no tests reported)" diff --git a/composer/foundry/state.py b/composer/foundry/state.py index 60928753..c7fb20e4 100644 --- a/composer/foundry/state.py +++ b/composer/foundry/state.py @@ -1,9 +1,8 @@ """State types + completion gate for the foundry test author. -Mirrors ``composer/spec/cvl_generation.py`` for the foundry workflow: +The generic authoring state (:mod:`composer.authoring.state`) supplies the buffer, the skip list, +the validation stamps and the digest gate. What is foundry's own: -* ``curr_test: str | None`` — the buffered ``.t.sol`` source. Single - file, single buffer (per the design decision). * ``expected_failures: dict[str, str]`` — test-name → reason map for tests intentionally expected to fail. Populated by ``expect_test_failure``, cleared per-key by ``expect_test_passage``. The ``forge_test`` runner @@ -11,27 +10,25 @@ * ``last_test_names`` — the test-function names reported by the most recent ``forge_test`` run (parsed from forge's JSON output). The runner records this unconditionally on every run that produced parseable - results; the publish gate uses it to check the declared property→test - mapping against the tests that *actually ran*, rather than trusting the - agent's transcription. -* ``skipped`` / ``property_tests`` / ``validations`` / ``required_validations`` - — same shape as the CVL counterpart, just keyed against ``curr_test`` - for the digest. ``property_tests`` carries the property→test-function - mapping enforced at publish time via ``validate_property_tests``. + results; the publish gate uses it as the ground truth + :func:`~composer.authoring.state.validate_check_mapping` checks the declared property→test + mapping against, rather than trusting the agent's transcription. +* ``property_tests`` — that mapping. """ -import hashlib -from typing import Annotated, Callable, NotRequired -from typing_extensions import TypedDict +from typing import Annotated, NotRequired from langgraph.graph import MessagesState from pydantic import BaseModel, Field from graphcore.graph import FlowInput -from composer.core.state import merge_validation +from composer.authoring.state import ( + AuthoringExtra, MappingVocab, SkippedProperty, check_completion, + merge_expected_failures, validate_check_mapping, +) from composer.spec.context import CacheKey, FoundryGeneration, FoundryJudge -from composer.spec.cvl_generation import SkippedProperty, _merge_skips +from composer.spec.types import CheckName, PropertyTitle FORGE_TEST_VALIDATION_KEY = "forge_test" @@ -42,45 +39,30 @@ # namespace and thread ids). FOUNDRY_JUDGE_KEY = CacheKey[FoundryGeneration, FoundryJudge]("judge") -class FoundryTestExtra(TypedDict): - curr_test: str | None - class PropertyTestMapping(BaseModel): """Maps one property from the batch to the foundry test function(s) that demonstrate it.""" - property_title: str = Field( + property_title: PropertyTitle = Field( description="The unique snake_case title of the property (from the " "batch listing) that these tests demonstrate" ) - tests: list[str] = Field( + tests: list[CheckName] = Field( description="The names of the test functions (``test_*`` / " "``testFuzz_*`` / ``invariant_*``) in the test file that demonstrate " "this property" ) -def _merge_expected_failures(left: dict[str, str], right: dict[str, str]) -> dict[str, str]: - """An empty reason removes the marking — ``expect_test_failure`` rejects - empty reasons at the tool boundary, so an empty value can only mean - ``expect_test_passage``'s delete.""" - to_ret = left.copy() - for k, v in right.items(): - if not v: - to_ret.pop(k, None) - continue - to_ret[k] = v - return to_ret - - -class FoundryGenerationExtra(FoundryTestExtra): - skipped: Annotated[list[SkippedProperty], _merge_skips] +class FoundryGenerationExtra(AuthoringExtra): property_tests: list[PropertyTestMapping] - validations: Annotated[dict[str, str], merge_validation] - required_validations: list[str] - expected_failures: Annotated[dict[str, str], _merge_expected_failures] - last_test_names: list[str] | None + expected_failures: Annotated[dict[CheckName, str], merge_expected_failures] + last_test_names: list[CheckName] | None failed: bool | None + #: Stamped True by the budget monitor's state transformer when the wrap-up alert fires (the + #: same update that lifts the validation gates), so the published result is known to be a + #: budget-curtailed partial rather than a validated delivery. + budget_curtailed: bool class FoundryGenerationInput(FoundryGenerationExtra, FlowInput): @@ -91,106 +73,32 @@ class FoundryGenerationState(FoundryGenerationExtra, MessagesState): result: NotRequired[str] -def _foundry_digest(curr_test: str, skipped: list[SkippedProperty]) -> str: - """Stable digest of the publish surface — the buffered test source plus - the skip declarations. Stamps from ``forge_test`` use this; a subsequent - ``put_test_raw`` invalidates them by changing ``curr_test``.""" - h = hashlib.md5() - h.update(curr_test.encode()) - for s in skipped: - h.update(f"{s.property_title}:{s.reason}".encode()) - return h.hexdigest() - - -def make_foundry_validation_stamper( - key: str, -) -> Callable[[FoundryGenerationExtra], dict[str, str]]: - def stamp(state: FoundryGenerationExtra) -> dict[str, str]: - return { - key: _foundry_digest(state["curr_test"] or "", state["skipped"]) - } - return stamp +def check_foundry_completion(state: FoundryGenerationExtra) -> str | None: + """Return None if the publish gate is satisfied, otherwise the reason.""" + return check_completion(state, nothing_written="no test written yet.") -def check_foundry_completion(state: FoundryGenerationExtra) -> str | None: - """Return None if the publish gate is satisfied, otherwise the reason. - - Required validations must have stamps whose digest matches the current - ``curr_test + skipped`` digest. A stamp that doesn't match is treated as - stale (the agent edited the test after the stamp was issued).""" - test = state["curr_test"] - if test is None: - return "Completion REJECTED: no test written yet." - digest = _foundry_digest(test, state["skipped"]) - validations = state["validations"] - for key in state["required_validations"]: - if validations.get(key) != digest: - return ( - f"Completion REJECTED: {key} validation not satisfied or stale." - ) - return None +#: How the foundry author words its publish-time mapping. Unlike CVL, forge names every test it +#: ran, so the mapping is checked against that ground truth in both directions. +_FOUNDRY_MAPPING = MappingVocab( + check_noun="test", + field_name="property_tests", + ran_source="the stamping forge_test invocation", +) def validate_property_tests( property_tests: list[PropertyTestMapping], skipped: list[SkippedProperty], - titles: list[str], - ran_test_names: list[str], + titles: list[PropertyTitle], + ran_test_names: list[CheckName], ) -> str | None: - """Validate the property→tests mapping declared at completion time. - - Unlike the CVL counterpart (which has to trust the agent's transcription - of rule names), forge reports the name of every test it ran, so the - mapping is checked against ground truth in both directions: every test - name in the mapping must have actually run, and every test that ran must - be tied back to some property. Plus the usual coverage checks: every - non-skipped property (referenced by its unique title) maps to at least - one test, no skipped property is mapped, every referenced title exists, - and no title is mapped twice. - """ - valid_titles = set(titles) - skipped_titles = {s.property_title for s in skipped} - ran = set(ran_test_names) - errors: list[str] = [] - mapped: set[str] = set() - claimed_tests: set[str] = set() - for m in property_tests: - if m.property_title not in valid_titles: - errors.append(f"Unknown property title {m.property_title!r} (not one of the batch's properties).") - continue - if m.property_title in mapped: - errors.append(f"Property {m.property_title!r} appears more than once in the mapping.") - continue - mapped.add(m.property_title) - if m.property_title in skipped_titles: - errors.append( - f"Property {m.property_title!r} is marked as skipped and must not appear " - "in the mapping (un-skip it or remove it)." - ) - continue - names = [t.strip() for t in m.tests if t.strip()] - if not names: - errors.append(f"Property {m.property_title!r} must map to at least one non-empty test name.") - continue - for t in names: - claimed_tests.add(t) - if t not in ran: - errors.append( - f"Property {m.property_title!r} claims test {t!r}, but no test by that " - "name ran in the stamping forge_test invocation." - ) - for t in titles: - if t in skipped_titles or t in mapped: - continue - errors.append(f"Property {t!r} is neither skipped nor mapped to any tests.") - for t in sorted(ran - claimed_tests): - errors.append( - f"Test {t!r} ran but is not tied back to any property in the mapping. " - "Every test in the file must demonstrate one of the batch's properties." - ) - if errors: - return ( - "Completion REJECTED: the property_tests mapping is invalid. Fix all of the " - "following and resubmit:\n- " + "\n- ".join(errors) - ) - return None + """Validate the property→tests mapping declared at completion time, against the tests forge + actually ran.""" + return validate_check_mapping( + [(m.property_title, m.tests) for m in property_tests], + skipped, + titles, + _FOUNDRY_MAPPING, + ran=ran_test_names, + ) diff --git a/composer/input/files.py b/composer/input/files.py index 6a5d3569..ce79c486 100644 --- a/composer/input/files.py +++ b/composer/input/files.py @@ -28,7 +28,7 @@ import zlib from abc import ABC, abstractmethod from dataclasses import dataclass -from typing import Protocol, overload +from typing import Protocol, Sequence, overload from composer.llm.types import CacheLevel @@ -95,6 +95,67 @@ def file_digest(path: pathlib.Path) -> str: return _bytes_digest(path.read_bytes()) +DOCUMENT_SUFFIXES: frozenset[str] = frozenset({ + ".md", ".markdown", ".txt", ".rst", ".pdf", +}) +"""What :func:`discover_documents` collects. Narrower than what +:meth:`FileUploader.get_document` accepts, so a sweep does not inhale contract source +sitting beside the docs; an explicitly-named path is not filtered by it.""" + + +def discover_documents(root: pathlib.Path) -> list[pathlib.Path]: + """The :data:`DOCUMENT_SUFFIXES` files directly in ``root``, sorted by name; not + recursive, hidden files skipped. The order feeds a cache key, so the same directory + must always yield the same list. Raises ``ValueError`` if ``root`` is not a + directory; an empty directory yields an empty list.""" + if not root.is_dir(): + raise ValueError(f"not a directory: {root}") + return sorted( + ( + # Name checks before is_file so the stat only runs for qualifying names. + p for p in root.iterdir() + if p.suffix.lower() in DOCUMENT_SUFFIXES + and not p.name.startswith(".") + and p.is_file() + ), + key=lambda p: p.name, + ) + + +def resolve_document_paths(paths: Sequence[str] | None) -> list[pathlib.Path]: + """Resolve extra-context selections into an ordered document list. Each entry is + either a document, taken as-is, or a directory, swept by :func:`discover_documents` + and spliced in place. + + Order is fully determined — entries in the order given, each sweep sorted by name — + because the result feeds both the prompt and a cache key. Raises ``ValueError`` for a + path that does not exist, or a swept directory with no supported documents. + + Entries are used exactly as given: a relative one is interpreted against the process + working directory (*not* the pipeline's ``--project-root``), and paths come back in + the form they went in, so a swept directory yields relative paths iff the directory + entry was relative.""" + if not paths: + return [] + + out: list[pathlib.Path] = [] + for raw in paths: + p = pathlib.Path(raw) + if p.is_dir(): + found = discover_documents(p) + if not found: + raise ValueError( + f"no supported documents directly under {p} (looked for " + f"{', '.join(sorted(DOCUMENT_SUFFIXES))}; the sweep is not recursive)" + ) + out.extend(found) + elif p.is_file(): + out.append(p) + else: + raise ValueError(f"no such file or directory: {p}") + return out + + # Suffixes treated as binary regardless of byte content; short-circuits # the byte-scan heuristic for common cases. _KNOWN_BINARY_SUFFIXES = {".pdf"} diff --git a/composer/input/parsing.py b/composer/input/parsing.py index a774b74b..fee48832 100644 --- a/composer/input/parsing.py +++ b/composer/input/parsing.py @@ -1,7 +1,22 @@ import argparse from typing import TypeVar, Protocol, cast, Annotated, get_type_hints, get_origin, Any, get_args, Union -from composer.input.types import CommandLineArgs, ResumeArgs, Arg, OptionalArg, RAGDBOptions, ModelOptions, LanggraphOptions, UploadPaths, InputData -from composer.input.files import FileUploader +from composer.input.types import CommandLineArgs, ResumeArgs, Arg, OptionalArg, RAGDBOptions, ModelOptions, LanggraphOptions, UploadPaths, InputData, SpecInput +from composer.input.files import DOCUMENT_SUFFIXES, FileUploader + + +def add_extra_context_args(parser: argparse.ArgumentParser) -> None: + """Register ``--extra-context``. Shared so the entry points cannot drift: + ``cache-autoprove inputs`` rebuilds a run's bug-analysis key from this same + attribute.""" + parser.add_argument( + "--extra-context", type=str, action="append", default=None, metavar="PATH", + help="Path to a document (text or pdf) of any extra information about the " + "application — protocol notes, deployment assumptions, audit scope — to hand " + "to the property extraction process as background. A directory is swept for " + f"every {'/'.join(sorted(DOCUMENT_SUFFIXES))} directly in it (not recursive, " + "hidden files skipped). Repeat the flag for several; they reach the prompt in " + "the order given.", + ) ArgNS = TypeVar("ArgNS", covariant=True) @@ -148,7 +163,13 @@ async def upload_input(uploader: FileUploader, args: UploadPaths) -> InputData: system_doc = await uploader.get_document(args.system_doc) if system_doc is None: raise FileNotFoundError(f"System document not found or not a file: {args.system_doc}") - return InputData(spec=spec, system_doc=system_doc, intf=intf) + # The legacy CLI triad is single-spec; map it to a one-element specs list at + # the conventional codegen path. The pipeline is plumbed for N specs. + return InputData( + specs=[SpecInput(file=spec, vfs_path="rules.spec")], + system_doc=system_doc, + intf=intf, + ) def _common_resume_args(parser: argparse.ArgumentParser) -> None: diff --git a/composer/input/types.py b/composer/input/types.py index 602ea2ad..c3eac6d1 100644 --- a/composer/input/types.py +++ b/composer/input/types.py @@ -72,7 +72,6 @@ class LanggraphOptions(Protocol): default=DEFAULT_RECURSION_LIMIT )] - class WorkflowOptions(RAGDBOptions, LanggraphOptions, Protocol): prover_capture_output: bool prover_keep_folders: bool @@ -171,16 +170,32 @@ class ResumeArgs(WorkflowOptions, ModelOptions, Protocol): working_dir: str +@dataclass +class SpecInput: + """A single spec file paired with the VFS path at which it should be + materialized inside the workflow's virtual filesystem.""" + file: TextDocument + vfs_path: str + + @dataclass class InputData: + """Normalized codegen workflow input for a single contract task. + + Carries one or more specs (all describing the same contract), the + contract's interface, and the surrounding system document. Specs and + interface are guaranteed text; system_doc may be PDF or text. Each spec's + VFS path is resolved at load time so downstream code (executor, prover + tool, audit) can key off a stable location. """ - Represents all of the file inputs provided by the user after loading. - Spec and interface are guaranteed text; system_doc may be PDF or text. - """ - spec: TextDocument + specs: list[SpecInput] system_doc: Document intf: TextDocument + @property + def spec_vfs_paths(self) -> list[str]: + return [s.vfs_path for s in self.specs] + class ResumeInput(Protocol): @property @@ -198,7 +213,10 @@ def thread_id(self) -> str: @dataclass class ResumeIdData: thread_id: str - new_spec: TextNativeFS + # VFS path → new spec content. Only paths present here are updated on + # resume; other specs keep their prior state. Single-spec CLI resumes + # carry one entry. + new_specs: dict[str, TextNativeFS] comments: Optional[str] new_system: Optional[NativeFS] diff --git a/composer/io/context.py b/composer/io/context.py index c2fa1ba3..e15162d0 100644 --- a/composer/io/context.py +++ b/composer/io/context.py @@ -23,16 +23,18 @@ peels these layers to reconstruct the full execution path. """ -from contextvars import ContextVar +from abc import ABC, abstractmethod +from contextvars import ContextVar, Token from contextlib import asynccontextmanager import asyncio + from composer.io.protocol import IOHandler from composer.io.stream import EventQueue from composer.io.event_handler import EventHandler -from typing import Any, Mapping +from typing import Any, Awaitable, Callable, Mapping, Protocol, cast, override from composer.io.events import ( AllEvents, InnerEvent, Nested, NextCheckpoint, @@ -46,7 +48,7 @@ from langchain_core.runnables import RunnableConfig from composer.io.graph_runner import SinkProtocol, run_graph as _run_graph - +from langgraph._internal._typing import StateLike _io_handler : ContextVar[None | tuple[EventQueue, IOHandler[Any], EventHandler]] = ContextVar("_io_handler", default=None) @@ -138,6 +140,7 @@ def emit_custom_event(payload: Mapping[str, Any]): raise ValueError("No IO handler installed") curr_io[0].push(ProgressEvent(dict(payload))) + async def run_graph[S: StateLike, C: StateLike | None, I: StateLike]( graph: CompiledStateGraph[S, C, I, Any], ctxt: C, @@ -145,6 +148,7 @@ async def run_graph[S: StateLike, C: StateLike | None, I: StateLike]( run_conf: RunnableConfig, description: str, within_tool: str | None = None, + retry: "RetryPolicy | FreshRetryPolicy[S, I] | None" = None, ) -> S: """Execute a graph within the current ``with_handler`` scope. @@ -154,6 +158,15 @@ async def run_graph[S: StateLike, C: StateLike | None, I: StateLike]( the drainer can reconstruct the execution path. HITL interrupts are bridged to ``IOHandler.human_interaction()``. + + Retry is first-class: the floor policy (``retry`` when it is a + :class:`RetryPolicy`, else the ambient :func:`install_retry_policy` one) + re-runs transient failures from the last checkpoint this run streamed; a + :class:`FreshRetryPolicy` (``retry`` when it is one) escalates wedged + failures by rebuilding the input on a fresh thread. Checkpoints are + tracked by wrapping THIS run's sink — children's checkpoints arrive + ``Nested``-wrapped and don't register — so retry works for nested + (sub-agent) runs, each tracking only its own thread. """ curr_io = _io_handler.get() if curr_io is None: @@ -162,7 +175,8 @@ async def run_graph[S: StateLike, C: StateLike | None, I: StateLike]( (ev, handle, _) = curr_io # Determine thread_id from config - tid = run_conf.get("configurable", {}).get("thread_id") + configurable = run_conf.get("configurable", {}) + tid = configurable.get("thread_id") if tid is None: raise ValueError("thread_id required in run config") @@ -174,7 +188,20 @@ async def run_graph[S: StateLike, C: StateLike | None, I: StateLike]( (parent_sink, parent_tid) = parent sink = lambda event: parent_sink(Nested(event, parent_id=parent_tid)) - tok = _current_sink.set((sink, tid)) + # The most recent checkpoint THIS run committed (seeded from an explicit + # resume point when the caller passed one) — the resume anchor for floor + # retries. Recorded by wrapping this run's own sink rather than the + # scope's queue, so concurrent and nested runs never cross-talk. + last_checkpoint: str | None = configurable.get("checkpoint_id") + + def tracking_sink(event: GraphEvents) -> None: + nonlocal last_checkpoint + if isinstance(event, NextCheckpoint): + last_checkpoint = event.checkpoint_id + sink(event) + + floor = retry if isinstance(retry, RetryPolicy) else _run_retry_policy.get() + fresh = retry if isinstance(retry, FreshRetryPolicy) else None async def handle_human( h: Any, @@ -182,16 +209,201 @@ async def handle_human( ) -> str: return await handle.human_interaction(h, lambda: None) - try: - return await _run_graph( - event_sink=sink, - graph=graph, - ctxt=ctxt, - input=input, - run_conf=run_conf, - description=description, - human_handler=handle_human, - within_tool=within_tool, - ) - finally: - _current_sink.reset(tok) + async def _attempt(inp: I, tid_: str, desc: str) -> S: + conf = run_conf.copy() + merged: dict[str, Any] = {**configurable, "thread_id": tid_} + # ``configurable`` may carry the caller's original resume point; the + # tracked checkpoint (None after a fresh restart) is authoritative. + merged.pop("checkpoint_id", None) + if last_checkpoint is not None: + merged["checkpoint_id"] = last_checkpoint + conf["configurable"] = merged + tok = _current_sink.set((tracking_sink, tid_)) + try: + return await _run_graph( + event_sink=tracking_sink, + graph=graph, + ctxt=ctxt, + input=inp, + run_conf=conf, + description=desc, + human_handler=handle_human, + within_tool=within_tool, + ) + finally: + _current_sink.reset(tok) + + async def _with_floor(inp: I, tid_: str, desc_base: str) -> S: + attempts = floor.max_retries if floor is not None else 1 + for i in range(attempts): + desc = desc_base if i == 0 else f"{desc_base} (Retry {i})" + try: + return await _attempt(inp, tid_, desc) + except Exception as e: + # Non-retryable, or that was the last attempt: propagate as-is + # (no parting backoff — there is no next attempt to wait for). + if floor is None or not floor.should_retry(e) or i + 1 >= attempts: + raise e + if last_checkpoint is None: + raise ValueError("Got retryable error but no checkpoint to resume from") from e + await floor.try_backoff(i) + assert False # unreachable: the last iteration either returns or raises + + if fresh is None: + return await _with_floor(input, tid, description) + + curr_input = input + curr_tid = tid + for j in range(fresh.max_fresh_retries): + if j > 0: + last_checkpoint = None + desc = description if j == 0 else f"{description} (Attempt {j})" + try: + return await _with_floor(curr_input, curr_tid, desc) + except Exception as e: + # Not fresh-retryable, or that was the last fresh attempt: + # propagate as-is (no pointless rebuild of an input that will + # never run). + if not fresh.should_retry_fresh(e) or j + 1 >= fresh.max_fresh_retries: + raise e + if last_checkpoint is None: + raise ValueError("Have retryable error but no checkpoint from which to recover") from e + last_state = await graph.aget_state({"configurable": { + "thread_id": curr_tid, + "checkpoint_id": last_checkpoint + }}) + if not last_state.values: + raise ValueError(f"No state at the last checkpoint {last_checkpoint} in {curr_tid}") from e + (curr_input, curr_tid) = await fresh.rebuild_input( + last_state=cast(S, last_state.values), + last_input=curr_input + ) + assert False # unreachable: the last iteration either returns or raises + + +async def run_to_completion[I: StateLike, S: StateLike, C: StateLike | None]( + graph: CompiledStateGraph[S, C, I, Any], + input: I, + thread_id: str, + context: C = None, + *, + checkpoint_id: str | None = None, + recursion_limit: int, + description: str, + within_tool: str | None = None, + retry: "RetryPolicy | FreshRetryPolicy[S, I] | None" = None, +) -> S: + """Run a compiled state graph to completion. + + Delegates to :func:`run_graph`, which handles event nesting automatically + via context vars and applies the ambient (or ``retry``-overridden) retry + policy. Requires ``with_handler()`` to be active. + + ``within_tool`` is the calling tool's ``tool_call_id`` when this graph is + being run as a sub-agent from inside a tool. It anchors the sub-graph's + UI panel under the tool-call widget so the renderer can mount nested + output in the right place. Pass ``self.tool_call_id`` from a tool that + mixes in ``WithInjectedId``; leave ``None`` for top-level / pipeline- + phase invocations. + """ + run_conf: RunnableConfig = { + "configurable": {"thread_id": thread_id}, + "recursion_limit": recursion_limit, + } + if checkpoint_id is not None: + run_conf["configurable"]["checkpoint_id"] = checkpoint_id + + return await run_graph( + graph=graph, + ctxt=context, + input=input, + run_conf=run_conf, + description=description, + within_tool=within_tool, + retry=retry, + ) + +class RetryPolicy(ABC): + """The transient-failure ("floor") retry contract: which exceptions are + worth re-running from the last checkpoint, and how long to back off + between attempts. Installed run-wide via :func:`install_retry_policy`, + or passed per-run through ``run_graph`` / ``run_to_completion`` to + override the ambient floor.""" + + max_retries: int + + @abstractmethod + def should_retry(self, exc: Exception) -> bool: ... + + @abstractmethod + async def try_backoff(self, try_count: int): ... + + +class FreshRetryPolicy[S: StateLike, I: StateLike](ABC): + """Fresh-start escalation, independent of the floor policy: for failures + a checkpoint-resume can't fix (the thread is wedged, not the request), + rebuild the input from the crashed attempt's last checkpointed state and + start over on the fresh thread id ``rebuild_input`` returns. + + Deliberately NOT a :class:`RetryPolicy`: specifying an escalation does not + require restating a floor — a plain ``FreshRetryPolicy`` rides whatever + floor is ambiently installed. A type inheriting BOTH overrides the floor + as well.""" + + max_fresh_retries: int + + @abstractmethod + def should_retry_fresh(self, exc: Exception) -> bool: ... + + @abstractmethod + async def rebuild_input( + self, last_state: S, last_input: I + ) -> tuple[I, str]: ... + + +type RetryPredicate = Callable[[Exception], bool] + +type Backoff = Callable[[int], Awaitable[None]] + +async def exponential_backoff( + i: int +): + await asyncio.sleep((2 * 2 ** i) * 60) # aggressive backoff in *minutes* + +DEFAULT_MAX_RETRIES = 3 + +class DefaultRetryPolicy(RetryPolicy): + def __init__( + self, + should_retry: RetryPredicate, + backoff: Backoff = exponential_backoff, + max_retries: int = DEFAULT_MAX_RETRIES, + ): + self._should_retry = should_retry + self.backoff_policy = backoff + self.max_retries = max_retries + + @override + def should_retry(self, exc: Exception) -> bool: + return self._should_retry(exc) + + @override + async def try_backoff(self, try_count: int): + await self.backoff_policy(try_count) + + +_run_retry_policy: ContextVar[RetryPolicy | None] = ContextVar("_run_retry_policy", default=None) +"""The run-wide retry floor. Read by every ``run_graph`` (nested sub-agent +runs included) when no per-run policy overrides it; ``None`` means failures +propagate on the first attempt, as before.""" + + +def install_retry_policy(policy: RetryPolicy | None) -> Token[RetryPolicy | None]: + """Install the run-wide retry floor. + + The pipeline harness calls this once per run (mirroring + ``install_run_summary``) before any graph work spawns; contextvar + inheritance carries it into every task of the run. Returns the token so + scoped callers (tests) can reset.""" + return _run_retry_policy.set(policy) + diff --git a/composer/io/task_host.py b/composer/io/task_host.py new file mode 100644 index 00000000..c020944b --- /dev/null +++ b/composer/io/task_host.py @@ -0,0 +1,64 @@ +from typing import Literal, Callable, Awaitable +from uuid import uuid4 +import asyncio +from dataclasses import dataclass + +@dataclass +class _TaskHandle[T]: + desc: str + handle: asyncio.Future[T] + event: asyncio.Event + ty: type[T] + +@dataclass +class TaskStatus: + desc: str + status: Literal["running", "complete"] + +class NoSuchTaskError(RuntimeError): + ... + +class TaskHost: + def __init__(self): + self._task_map : dict[str, _TaskHandle]= {} + self._lock = asyncio.Lock() + + async def list_tasks(self) -> dict[str, TaskStatus]: + return { + k: TaskStatus( + desc=j.desc, + status="running" if not j.handle.done() else "complete" + ) for (k, j) in self._task_map.items() + } + + async def launch[T](self, description: str, t: type[T], task: Callable[[], Awaitable[T]]) -> str: + # Callable[[], Awaitable[T]] isn't a "coroutine like" for the asyncio typing, so make that explicit + async def task_wrap(): + return await task() + async with self._lock: + t_id = uuid4().hex + compl = asyncio.Event() + task_fut = asyncio.create_task( + task_wrap() + ) + task_fut.add_done_callback(lambda _: compl.set()) + self._task_map[t_id] = _TaskHandle( + desc=description, + handle=task_fut, + event=compl, + ty=t + ) + return t_id + + async def await_result[T](self, task_id: str, t: type[T]) -> T: + async with self._lock: + handle = self._task_map.get(task_id) + if handle is None: + raise NoSuchTaskError() + assert handle.ty is t, "Type mismatch" + await handle.event.wait() + res = handle.handle.result() + async with self._lock: + if task_id in self._task_map: + del self._task_map[task_id] + return res diff --git a/composer/io/task_tools.py b/composer/io/task_tools.py new file mode 100644 index 00000000..acde2b4a --- /dev/null +++ b/composer/io/task_tools.py @@ -0,0 +1,59 @@ +"""The author-side surface of the run's :class:`TaskHost`: list the background +tasks plugin tools have launched, and collect their reports. Task results are +always strings — the lingua franca of tool results — so ``RetrieveTask`` is +the single retrieval tool regardless of which plugin launched the task. + +Bind both over the host the author owns:: + + TaskListTool.bind(host).as_tool(TASK_LIST) + RetrieveTask.bind(host).as_tool(RETRIEVE_TASK) +""" + +from typing import override + +from pydantic import Field + +from graphcore.tools.schemas import WithAsyncDependencies + +from .task_host import NoSuchTaskError, TaskHost + +TASK_LIST = "task_list" +RETRIEVE_TASK = "retrieve_task" + + +class TaskListTool(WithAsyncDependencies[str, TaskHost]): + """ + List the background tasks launched in this session: each task's ID, what + it is doing, and whether it is still running or has a report ready. + """ + + @override + async def run(self) -> str: + with self.tool_deps() as host: + tasks = await host.list_tasks() + if not tasks: + return "No background tasks are outstanding (none launched, or every report already retrieved)." + return "\n".join( + f"{task_id}: {info.desc} [{info.status}]" + for (task_id, info) in tasks.items() + ) + + +class RetrieveTask(WithAsyncDependencies[str, TaskHost]): + """ + Collect a background task's report by ID, waiting for the task to finish + if it is still running. Each report can be retrieved exactly once. + """ + task_id: str = Field(description="The task ID reported when the task was launched") + + @override + async def run(self) -> str: + with self.tool_deps() as host: + try: + return await host.await_result(self.task_id, str) + except NoSuchTaskError: + return ( + f"No task with ID {self.task_id}: it never existed, or its " + "report was already retrieved (reports are single-shot); " + f"see `{TASK_LIST}`." + ) diff --git a/composer/io/thread_logging.py b/composer/io/thread_logging.py index 37a1cb35..9643e589 100644 --- a/composer/io/thread_logging.py +++ b/composer/io/thread_logging.py @@ -1,4 +1,4 @@ -from typing import TypedDict, Protocol, AsyncIterator, Any, Callable +from typing import NotRequired, TypedDict, Protocol, AsyncIterator, Any, Callable from dataclasses import dataclass from datetime import datetime, UTC from uuid import uuid4 @@ -8,6 +8,7 @@ from langchain_core.runnables import RunnableConfig from langgraph.store.base import BaseStore from composer.core.user import user_data_ns +from composer.diagnostics.budget import current_cost_center class _WithTimings(TypedDict): start_time: str @@ -25,6 +26,12 @@ class ThreadMeta(_WithTimings): start_checkpoint_id: str | None end_checkpoint_id: str | None + #: The named budget scope this thread ran under (a `PhaseBudget` phase name), or + #: None for work outside any named scope (the run pool / pre-pipeline). Recorded + #: whether or not a budget was installed. NotRequired: absent on records written + #: before cost-center tracking existed. + cost_center: NotRequired[str | None] + DEFAULT_META_NS = ("logging",) def runs_ns(parent_ns: tuple[str, ...]) -> tuple[str, ...]: @@ -96,7 +103,10 @@ async def start( "run_id": self.run_id, "start_checkpoint_id": start_checkpoint, "thread_id": thread_id, - "description": description + "description": description, + # Read from the ambient contextvar: start() runs (via log_thread) in the + # thread's own task, inside whatever named budget scope spawned it. + "cost_center": current_cost_center() } try: await self.store.aput( diff --git a/composer/llm/anthropic.py b/composer/llm/anthropic.py index 854e5143..b1101fcf 100644 --- a/composer/llm/anthropic.py +++ b/composer/llm/anthropic.py @@ -14,6 +14,7 @@ from composer.llm.provider import ( ProviderServiceBase, ProviderSpec, compaction_threshold ) +from composer.llm.pricing import PriceProvider, price_provider_for from .types import CacheLevel from .list_iter import ListIter, NoSuchElementError @@ -218,6 +219,19 @@ def cache_marker(self, payload: "RawMessageType", cache_level: CacheLevel) -> "R } return to_ret + @override + def should_retry(self, exc: Exception) -> bool: + """Mirrors the SDK's own ``_should_retry`` status roster (408/409/429 + and every 5xx, which covers 529 overloaded) plus connection-level + failures (``APITimeoutError`` subclasses ``APIConnectionError``). + 400-class request errors are deterministic — an over-long prompt fails + identically on every attempt — and are deliberately excluded.""" + if isinstance(exc, anthropic.APIConnectionError): + return True + if isinstance(exc, anthropic.APIStatusError): + return exc.status_code in (408, 409, 429) or exc.status_code >= 500 + return False + @dataclass class AnthropicModelProvider: """``ModelProvider`` for Anthropic. Probes ``model_name`` once at @@ -227,15 +241,18 @@ class AnthropicModelProvider: model_name: str options: ModelConfiguration features: ModelFeatures + price_provider: PriceProvider provider: AnthropicService = field(default_factory=_get_service) + @staticmethod def create(model_name: str, options: ModelConfiguration) -> "AnthropicModelProvider": return AnthropicModelProvider( model_name, options, _model_parser(model_name), + price_provider_for(model_name) ) @property @@ -247,6 +264,7 @@ def builder_for( ) -> "BaseChatModel": from langchain_anthropic import ChatAnthropic from composer.diagnostics.usage_callback import UsageCallback + from composer.diagnostics.cost_callback import CostAccumulator opts = self.options thinking: dict[str, Any] | None @@ -281,7 +299,12 @@ def builder_for( betas=betas, thinking=thinking, model_kwargs=model_kwargs, - callbacks=[UsageCallback()], + callbacks=[ + UsageCallback(), + CostAccumulator( + self.price_provider, long_cache=cache_level == CacheLevel.LONG + ), + ], ) ANTHROPIC_SPEC = ProviderSpec( diff --git a/composer/llm/openai.py b/composer/llm/openai.py index 4d064d78..ae7619f3 100644 --- a/composer/llm/openai.py +++ b/composer/llm/openai.py @@ -21,9 +21,10 @@ from composer.input.files import UploaderBase, ContentRenderer from composer.input.types import ModelConfiguration -from composer.llm.provider import ( +from .provider import ( ProviderServiceBase, ProviderSpec, compaction_threshold ) +from .pricing import PriceProvider, price_provider_for from .types import CacheLevel from .list_iter import ListIter, NoSuchElementError @@ -150,6 +151,18 @@ def __init__(self): OpenAIFileUploader.lazy ) + @override + def should_retry(self, exc: Exception) -> bool: + """Same shape as the Anthropic mapping — the OpenAI SDK shares the + Stainless exception taxonomy: connection-level failures and + 408/409/429/5xx statuses are transient; 400-class request errors are + deterministic and excluded.""" + if isinstance(exc, openai.APIConnectionError): + return True + if isinstance(exc, openai.APIStatusError): + return exc.status_code in (408, 409, 429) or exc.status_code >= 500 + return False + @dataclass class OpenAIRenderer: def text_block(self, text: str, *, cache_level: CacheLevel = CacheLevel.NONE) -> dict: @@ -220,11 +233,14 @@ class OpenAIModelProvider: model_name: str options: ModelConfiguration features: OpenAIModelFeatures + price_provider: PriceProvider provider: OpenAIService = field(default_factory=_openai_service) @staticmethod def create(model_name: str, options: ModelConfiguration) -> "OpenAIModelProvider": - return OpenAIModelProvider(model_name, options, _model_parser(model_name)) + return OpenAIModelProvider( + model_name, options, _model_parser(model_name), price_provider_for(model_name) + ) @property def max_prompt_tokens(self) -> int: @@ -234,6 +250,8 @@ def builder_for( self, *, cache_level: CacheLevel = CacheLevel.NONE, disable_thinking: bool = False ) -> "BaseChatModel": from langchain_openai import ChatOpenAI + from composer.diagnostics.usage_callback import UsageCallback + from composer.diagnostics.cost_callback import CostAccumulator opts = self.options kwargs: dict[str, Any] = { @@ -253,6 +271,9 @@ def builder_for( max_completion_tokens=opts.tokens, timeout=None, max_retries=2, + # OpenAI has no cache-TTL knob, so long_cache stays False; cache_write_1h + # mirrors cache_write in the table anyway. + callbacks=[UsageCallback(), CostAccumulator(self.price_provider)], **kwargs, ) diff --git a/composer/llm/pricing.py b/composer/llm/pricing.py new file mode 100644 index 00000000..740ab991 --- /dev/null +++ b/composer/llm/pricing.py @@ -0,0 +1,152 @@ +from dataclasses import dataclass +from typing import Callable + +@dataclass(frozen=True) +class PriceTier: + """Per-million-token prices in USD for one model at one + context tier. + + ``input`` is the price for *fresh* input tokens (the bucket left + after subtracting ``cache_read`` and ``cache_write`` from the + total). ``output`` is the price for output tokens, which on the + OpenAI side already includes reasoning tokens (billed at the + output rate by both providers). ``cache_read`` is the + cache-hit rate. + + Cache writes come in two rates: ``cache_write`` is the 5-minute + ephemeral rate and ``cache_write_1h`` is the 1-hour ephemeral rate + (~2× base input on Anthropic). Which one applies depends on the + conversation's cache TTL. OpenAI has no separate cache-write rate, + so both fields carry the same value there.""" + input: float + output: float + cache_read: float + cache_write: float + cache_write_1h: float + + +@dataclass(frozen=True) +class ModelPricing: + """Pricing entry for one model family. ``long`` is the + long-context tier (used when input token count exceeds the + threshold) and applies only to OpenAI models that publish a + separate >272K-input rate; Anthropic models keep ``long = None`` + and bill everything at ``short`` rates.""" + short: PriceTier + long: PriceTier | None = None + + +# OpenAI's published >272K input-token threshold for long-context +# pricing. Once an individual call's input crosses this, the long +# tier applies *for the full session* per OpenAI's terms; we +# approximate that by switching on a per-message basis (a session +# that drifts above 272K will mostly stay there). +_OPENAI_LONG_CONTEXT_THRESHOLD = 272_000 + + +# Pricing tables transcribed from Anthropic + OpenAI rate cards. +# Sources should be re-checked when new model families ship or +# their costs update. +_PRICING: list[tuple[str, ModelPricing]] = [ + ("claude-fable-5", ModelPricing(short=PriceTier(10.00, 50.00, 1.00, 12.50, 20.0))), + + # claude-opus-5 and claude-opus-4.5 / 4.6 / 4.7 / 4.8 share a rate card; + # older 4 / 4.1 are pricier. Matching by prefix-of-prefix so + # "claude-opus-4-7" and "claude-opus-4-7-20260301" both hit the right entry. + ("claude-opus-5", ModelPricing(short=PriceTier(5.00, 25.00, 0.50, 6.25, 10.0))), + + # ---- Anthropic ---- + # claude-opus-4.5 / 4.6 / 4.7 share a rate card; older 4 / 4.1 + # are pricier. Matching by prefix-of-prefix so "claude-opus-4-7" + # and "claude-opus-4-7-20260301" both hit the right entry. + ("claude-opus-4-8", ModelPricing(short=PriceTier(5.00, 25.00, 0.50, 6.25, 10.00))), + ("claude-opus-4-7", ModelPricing(short=PriceTier(5.00, 25.00, 0.50, 6.25, 10.00))), + ("claude-opus-4-6", ModelPricing(short=PriceTier(5.00, 25.00, 0.50, 6.25, 10.00))), + ("claude-opus-4-5", ModelPricing(short=PriceTier(5.00, 25.00, 0.50, 6.25, 10.00))), + # o7 to retired models, they went to a datacenter upstate + # ("claude-opus-4-1", ModelPricing(short=PriceTier(15.00, 75.00, 1.50, 18.75, 30.00))), + # ("claude-opus-4", ModelPricing(short=PriceTier(15.00, 75.00, 1.50, 18.75, 30.00))), + + ("claude-sonnet-5", ModelPricing(short=PriceTier(3.00, 15.00, 0.30, 3.75, 6.00))), + + ("claude-sonnet-4-6", ModelPricing(short=PriceTier(3.00, 15.00, 0.30, 3.75, 6.00))), + ("claude-sonnet-4-5", ModelPricing(short=PriceTier(3.00, 15.00, 0.30, 3.75, 6.00))), + # ("claude-sonnet-4", ModelPricing(short=PriceTier(3.00, 15.00, 0.30, 3.75, 6.00))), + + ("claude-haiku-4-5", ModelPricing(short=PriceTier(1.00, 5.00, 0.10, 1.25, 2.00))), + + ("gpt-5.6-terra", ModelPricing( + short=PriceTier(2.00, 12.00, 0.20, 2.50, 2.50), + long=PriceTier(4.00, 18.00, 0.40, 5.00, 5.00), + )), + ("gpt-5.6-luna", ModelPricing( + short=PriceTier(0.20, 1.20, 0.02, 0.25, 0.25), + long=PriceTier(0.40, 1.80, 0.04, 0.50, 0.50), + )), + ("gpt-5.6", ModelPricing( + short=PriceTier(5.00, 30.00, 0.50, 6.25, 6.25), + long=PriceTier(10.00, 45.00, 1.00, 12.50, 12.50), + )), + + # ---- OpenAI ---- + # gpt-5.5 / 5.4 publish short (≤272K input) and long (>272K) tiers. + # Pro variants don't publish a cached-in discount (cache_read = + # base input). Mini/nano don't publish a long tier; we use short + # for everything on those. OpenAI has no separate cache-write rate, + # so cache_write_1h mirrors cache_write on every OpenAI entry. + ("gpt-5.5-pro", ModelPricing( + short=PriceTier(30.00, 180.00, 30.00, 30.00, 30.00), + long=PriceTier(60.00, 270.00, 60.00, 60.00, 60.00), + )), + ("gpt-5.5", ModelPricing( + short=PriceTier(5.00, 30.00, 0.50, 5.00, 5.00), + long=PriceTier(10.00, 45.00, 1.00, 10.00, 10.00), + )), + ("gpt-5.4-pro", ModelPricing( + short=PriceTier(30.00, 180.00, 30.00, 30.00, 30.00), + long=PriceTier(60.00, 270.00, 60.00, 60.00, 60.00), + )), + ("gpt-5.4-mini", ModelPricing(short=PriceTier(0.75, 4.50, 0.075, 0.75, 0.75))), + ("gpt-5.4-nano", ModelPricing(short=PriceTier(0.20, 1.25, 0.02, 0.20, 0.20))), + ("gpt-5.4", ModelPricing( + short=PriceTier(2.50, 15.00, 0.25, 2.50, 2.50), + long=PriceTier(5.00, 22.50, 0.50, 5.00, 5.00), + )), +] + + +def price_per_mtok(model: str | None, input_tokens: int) -> PriceTier | None: + """Look up per-MTok pricing by model name and call size. Returns + ``None`` for models with no table entry (cost contribution becomes + zero — better than guessing). + + Matched by prefix on the lowercased model name so dated revisions + (``claude-opus-4-7-20260301``, ``gpt-5.5-2026-...``) collapse into + the same family entry. Table is searched in order, so list more + specific prefixes (``gpt-5.5-pro``) before less specific + (``gpt-5.5``). For OpenAI models with a long tier, ``input_tokens`` + chooses short vs. long; Anthropic always uses the short tier.""" + if model is None: + return None + m = model.lower() + for prefix, pricing in _PRICING: + if m.startswith(prefix): + if pricing.long is not None and input_tokens > _OPENAI_LONG_CONTEXT_THRESHOLD: + return pricing.long + return pricing.short + return None + + +# A model's pricing curve as a function of a single call's input-token count: +# ``price_per_mtok`` with the model name curried away. The remaining argument +# picks the short/long context tier (OpenAI); Anthropic ignores it. +type PriceProvider = Callable[[int], PriceTier | None] + + +def price_provider_for(model: str | None) -> PriceProvider: + """Curry :func:`price_per_mtok` on ``model``: returns a callable mapping a + call's input-token count to its :class:`PriceTier` (or ``None`` when the + model has no table entry, so its cost contribution is zero).""" + def provider(input_tokens: int) -> PriceTier | None: + return price_per_mtok(model, input_tokens) + return provider diff --git a/composer/llm/provider.py b/composer/llm/provider.py index ce67fdaf..a77a110c 100644 --- a/composer/llm/provider.py +++ b/composer/llm/provider.py @@ -36,6 +36,14 @@ def uploader(self) -> FileUploader: def cache_marker(self, payload: "RawMessageType", cache_level: CacheLevel) -> "RawMessageType": ... + def should_retry(self, exc: Exception) -> bool: + """Whether ``exc`` is a transient provider-side failure worth retrying + (rate limits, overload, dropped connections) — as opposed to a + deterministic request error (an over-long prompt 400s identically + every time). The harness assembles this into the run-wide retry + policy (``composer.io.context.install_retry_policy``).""" + ... + class ProviderServiceBase(ABC): def __init__(self, mem_fact: Callable[["AsyncPostgresBackend"], "BaseTool"], @@ -59,6 +67,11 @@ def select_memory_tool( def cache_marker(self, payload: "RawMessageType", cache_level: CacheLevel) -> "RawMessageType": return payload + def should_retry(self, exc: Exception) -> bool: + # Providers opt in to retryability explicitly; unknown exceptions are + # never worth an automatic re-run. + return False + class ModelProvider(Protocol): """A provider-specific LLM backend, bound to one model. diff --git a/composer/natreq/extractor.py b/composer/natreq/extractor.py index 1febed76..0e227598 100644 --- a/composer/natreq/extractor.py +++ b/composer/natreq/extractor.py @@ -9,7 +9,6 @@ from graphcore.tools.results import result_tool_generator from langchain_core.tools import tool, BaseTool -from langchain_core.runnables import RunnableConfig from langchain_core.language_models.chat_models import BaseChatModel from langgraph.graph import MessagesState @@ -26,7 +25,7 @@ from composer.tools.thinking import RoughDraftState, get_rough_draft_tools from composer.templates.loader import load_jinja_template from composer.io.protocol import IOHandler -from composer.io.context import with_handler, run_graph +from composer.io.context import with_handler, run_to_completion from composer.io.event_handler import NullEventHandler from composer.ui.tool_display import tool_display from composer.diagnostics.timing import set_current_task_id @@ -120,7 +119,7 @@ async def get_requirements( options: RAGDBOptions, llm: BaseChatModel, sys_doc: Document, - spec_file: TextDocument, + specs: list[TextDocument], mem_tool: BaseTool, resume_artifact: ResumeArtifact | None, ) -> ExtractionResult: @@ -149,33 +148,40 @@ async def get_requirements( thread_id = uuid.uuid1().hex - config: RunnableConfig = {"configurable": {"thread_id": thread_id}} - sys_text = sys_doc.string_contents input_text : list[str | dict] = [ "The system document is as follows:", sys_text if sys_text is not None else sys_doc.to_dict(), - "The spec file is as follows:", - spec_file.string_contents + "The spec file(s) are as follows:", ] + for spec in specs: + input_text.append(spec.string_contents) if resume_artifact is not None: input_text.append(""" - You have previously performed this analysis on a prior version of the spec file. You have access to the + You have previously performed this analysis on a prior version of the spec file(s). You have access to the memories you generated during that prior analysis. Be sure to consult those memories to inform your analysis - of the system document. In addition, be sure to analyze the difference between the two specification files, - being sure to determine which natural language requirements are no longer needed (as they are now covered by the - spec). + of the system document. In addition, be sure to analyze the difference between the prior and current + specifications, being sure to determine which natural language requirements are no longer needed (as they are + now covered by the spec). """) - input_text.append("The OLD spec file is as follows:") - input_text.append( - resume_artifact.spec.contents - ) + input_text.append("The OLD spec file(s) are as follows:") + for path in resume_artifact.spec_vfs_paths: + prior = resume_artifact.spec_at(path) + if prior is not None: + input_text.append(prior.contents) graph_input = ExtractionInput(input=input_text, memory=None, did_read=False) async with with_handler(io, NullEventHandler()): # type: ignore[arg-type] with set_current_task_id(REQUIREMENTS_TASK_ID): - final_state = await run_graph(built, ExtractionContext(rag_db=db), graph_input, config, description="Requirements extraction") + final_state = await run_to_completion( + built, + graph_input, + thread_id=thread_id, + context=ExtractionContext(rag_db=db), + recursion_limit=250, + description="Requirements extraction", + ) assert "reqs" in final_state return ExtractionResult(reqs=final_state["reqs"], thread_id=thread_id) diff --git a/composer/natreq/judge.py b/composer/natreq/judge.py index af4ac253..5d793ec3 100644 --- a/composer/natreq/judge.py +++ b/composer/natreq/judge.py @@ -15,15 +15,14 @@ from langchain_core.tools import BaseTool, tool, InjectedToolCallId from langchain_core.language_models.chat_models import BaseChatModel from langchain_core.messages import ToolMessage -from langchain_core.runnables import RunnableConfig from langgraph.runtime import get_runtime from composer.templates.loader import load_jinja_template from composer.tools.thinking import RoughDraftState, get_rough_draft_tools from composer.core.state import AIComposerState -from composer.core.validation import reqs as req_key -from composer.core.context import AIComposerContext, compute_state_digest -from composer.io.context import run_graph +from composer.core.validation import ReqsValidation +from composer.core.context import AIComposerContext, stamp +from composer.io.context import run_to_completion from composer.ui.tool_display import tool_display class JudgeInput(FlowInput, RoughDraftState): @@ -150,12 +149,13 @@ async def requirements_evaluation( state: AIComposerState, tool_call_id: Annotated[str, InjectedToolCallId] ) -> Command | str: - judge_config: RunnableConfig = {"configurable": {"thread_id": uuid.uuid1().hex}} - judge_state = await run_graph( + judge_state = await run_to_completion( compiled_graph, - None, JudgeInput(input=[req_list], vfs=state["vfs"], orig_reqs=reqs, memory=None, did_read=False), - judge_config, + thread_id=uuid.uuid1().hex, + context=None, + # Preserves the langgraph default this run has always ridden on. + recursion_limit=250, description="Requirements evaluation", within_tool=tool_call_id ) @@ -171,16 +171,10 @@ async def requirements_evaluation( formatted_res = _format_result(judge_state["result"], skipped) if not all_satisfied: return formatted_res - digest = compute_state_digest( - c=get_runtime(AIComposerContext).context, - state=state - ) return Command(update={ "messages": [ ToolMessage(content=formatted_res, tool_call_id=tool_call_id) ], - "validation": { - req_key: digest - } + **stamp(ReqsValidation(), state) }) return requirements_evaluation diff --git a/composer/pipeline/cli.py b/composer/pipeline/cli.py index d0ece5fd..12c110ee 100644 --- a/composer/pipeline/cli.py +++ b/composer/pipeline/cli.py @@ -1,4 +1,5 @@ from typing import Protocol, AsyncIterator, TYPE_CHECKING +import json import sys import pathlib import enum @@ -7,9 +8,15 @@ import asyncio from dataclasses import dataclass +from pydantic import ( + BaseModel, ConfigDict, Field, NonNegativeFloat, PositiveFloat, ValidationError, + field_validator, +) + from composer.input.types import ( ExtendedModelOptions, ) +from composer.input.files import Document, resolve_document_paths from composer.diagnostics.logging_setup import setup_autoprove_logging from composer.spec.context import SourceFields, WorkflowContext, SourceCode @@ -17,7 +24,7 @@ from composer.workflow.services import IndexedConnections, standard_connections from composer.pipeline.ptypes import ( PipelineRun, BackendResult, - CorePipelineResult + CorePipelineResult, PhaseBudget, RunBudget ) from composer.spec.artifacts import ArtifactIdentifier from composer.spec.system_model import FeatureUnit, BaseApplication @@ -29,6 +36,7 @@ from .run_tags import AutoProveCacheTags, CACHE_ROOT_RECORD from composer.io.multi_job import HandlerFactory, run_task, TaskInfo from composer.diagnostics.timing import RunSummary, install_run_summary +from composer.io.context import DefaultRetryPolicy, install_retry_policy from composer.llm.registry import get_provider_for from composer.rag.models import get_model from composer.io.thread_logging import RunDataLogger, thread_logger, default_logging_ns @@ -72,6 +80,58 @@ def user_ns(*parts: str | tuple[str, ...]) -> tuple[str, ...]: return user_data_ns() + tuple(out) +BUDGET_PHASES: tuple[str, ...] = tuple(PhaseBudget.__annotations__) + + +class BudgetFile(BaseModel): + """Schema of a ``--budget`` file. + + A phase without an explicit cap defaults to ``total`` (bounded by the pool alone — + caps are ceilings, not allotments). A cap of ``0.0`` is legal and puts that phase in + the wrap-up window from its first monitor tick.""" + model_config = ConfigDict(extra="forbid") + + total: PositiveFloat = Field(description="The run pool in USD — the real bound on overall spend.") + caps: dict[str, NonNegativeFloat] = Field( + default_factory=dict, + description="Per-phase ceilings in USD, drawn against the pool; any subset of the phase names.", + ) + + @field_validator("caps") + @classmethod + def _known_phases(cls, v: dict[str, float]) -> dict[str, float]: + if (unknown := set(v) - set(BUDGET_PHASES)): + raise ValueError( + f"unknown phase(s) {sorted(unknown)}; valid phases: {list(BUDGET_PHASES)}" + ) + return v + + def to_run_budget(self) -> RunBudget: + return RunBudget( + total=self.total, + caps=PhaseBudget(**{p: self.caps.get(p, self.total) for p in BUDGET_PHASES}), # type: ignore[typeddict-item] + ) + + +def parse_budget_file(path: pathlib.Path) -> RunBudget: + """Parse a run-budget file (JSON, or YAML when PyYAML is installed) into a `RunBudget`. + See :class:`BudgetFile` for the schema.""" + if path.suffix in (".yaml", ".yml"): + try: + import yaml + except ImportError as e: + raise ValueError( + f"budget file {path} is YAML but PyYAML is not installed; use JSON instead" + ) from e + raw = yaml.safe_load(path.read_text()) + else: + raw = json.loads(path.read_text()) + try: + return BudgetFile.model_validate(raw).to_run_budget() + except ValidationError as e: + raise ValueError(f"invalid budget file {path}: {e}") from e + + class PipelineArgs(ExtendedModelOptions, Protocol): @property def recursion_limit(self) -> int: @@ -84,11 +144,22 @@ def interactive(self) -> bool: @property def max_concurrent(self) -> int: ... - + + @property + def max_cpu_tasks(self) -> int: + ... + @property def threat_model(self) -> str | None: ... + @property + def extra_context(self) -> list[str] | None: + """Documents of user-supplied background about the application, fed to property + inference in the order given. A directory entry is swept for the documents in + it.""" + ... + @property def cache_ns(self) -> str | None: ... @@ -108,11 +179,22 @@ def project_root(self) -> str: @property def main_contract(self) -> str: ... - + @property def system_doc(self) -> str | None: ... + @property + def budget(self) -> str | None: + """Path to a run-budget file (see :func:`parse_budget_file`), or None to run unbudgeted.""" + ... + + @property + def time_budget(self) -> float | None: + """ + Time in floating point seconds that autoprover should run. None to run with unlimited, in process timeout + """ + @dataclass class StagedPipeline: conns: IndexedConnections @@ -123,10 +205,10 @@ class StagedPipeline: root_key: str class Continuation[P: enum.Enum, H](Protocol): - async def __call__[FormT: BackendResult, A: ArtifactIdentifier, U: FeatureUnit, Main, App: BaseApplication]( + async def __call__[FormT: BackendResult, A: ArtifactIdentifier, U: FeatureUnit, Main, App: BaseApplication, Pre]( self, env: ServiceHost, - backend: PipelineBackend[P, FormT, H, A, U, Main, App], + backend: PipelineBackend[P, FormT, H, A, U, Main, App, Pre], ecosystem: Ecosystem[App, Main, U] ) -> CorePipelineResult[FormT]: ... @@ -152,6 +234,9 @@ async def cli_pipeline[P: enum.Enum, H]( project_root = pathlib.Path(args.project_root).resolve() main_contract_path, contract_name = args.main_contract.split(":", 1) + # Parse the budget up front so a malformed file fails before any services spin up. + budget = parse_budget_file(pathlib.Path(args.budget)) if args.budget is not None else None + full_contract_path = pathlib.Path(main_contract_path).resolve() if not full_contract_path.is_relative_to(project_root): raise ValueError(f"Invalid path: {full_contract_path} doesn't appear in project root {project_root}") @@ -162,12 +247,18 @@ async def cli_pipeline[P: enum.Enum, H]( tiered = get_provider_for(tiered=args) semaphore = asyncio.Semaphore(args.max_concurrent) + cpu_semaphore = asyncio.Semaphore(args.max_cpu_tasks) model = get_model() text_log, events_log = setup_autoprove_logging(project_root, thread_id) print(f"autoprove logs: {text_log}\n events: {events_log}", file=sys.stderr) print(f"Selected run id: {summary.run_id}") install_run_summary(summary) + # Run-wide retry floor: transient provider failures (as classified by the + # provider itself) resume any graph in the run from its last checkpoint + # instead of killing the whole pipeline. Installed once here — contextvar + # inheritance carries it into every task the run spawns. + install_retry_policy(DefaultRetryPolicy(tiered.provider_service.should_retry)) disc_cache_ns: tuple[str, ...] | None = ( user_ns(args.cache_ns, "discovery", @@ -251,6 +342,21 @@ async def cli_pipeline[P: enum.Enum, H]( await conns.uploader.get_document(pathlib.Path(threat_path)) if (threat_path := args.threat_model) is not None else None ) + # Gathered rather than awaited one at a time: a swept directory of PDFs is + # one upload round trip each. ``gather`` preserves order, which the prompt + # and the bug-analysis cache key both depend on. + context_paths = resolve_document_paths(args.extra_context) + loaded = await asyncio.gather(*(conns.uploader.get_document(p) for p in context_paths)) + extra_context: list[Document] = [] + for path, doc in zip(context_paths, loaded): + if doc is None: + raise ValueError(f"Fatal error, failed to read extra context: {path}") + extra_context.append(doc) + if budget is not None: + await data_logger("budget", { + "total": budget.total, "caps": dict(budget.caps), + }) + full_source = SourceCode( content=system_doc_content, contract_name=init_source.contract_name, @@ -259,9 +365,9 @@ async def cli_pipeline[P: enum.Enum, H]( relative_path=init_source.relative_path ) - async def cont[FormT: BackendResult, A: ArtifactIdentifier, U: FeatureUnit, Main, App: BaseApplication]( + async def cont[FormT: BackendResult, A: ArtifactIdentifier, U: FeatureUnit, Main, App: BaseApplication, Pre]( env: ServiceHost, - backend: PipelineBackend[P, FormT, H, A, U, Main, App], + backend: PipelineBackend[P, FormT, H, A, U, Main, App, Pre], ecosystem: Ecosystem[App, Main, U] ) -> CorePipelineResult[FormT]: await data_logger(CACHE_ROOT_RECORD, AutoProveCacheTags( @@ -270,6 +376,7 @@ async def cont[FormT: BackendResult, A: ArtifactIdentifier, U: FeatureUnit, Main memory_ns=memory_ns, plugins=applicable_plugin_manifest(ecosystem.unit_type), threat_model_digest=threat_model.to_digest() if threat_model is not None else None, + extra_context_digests=[d.to_digest() for d in extra_context], interactive=args.interactive, ).model_dump()) full_ctx = WorkflowContext.create( @@ -284,7 +391,8 @@ async def cont[FormT: BackendResult, A: ArtifactIdentifier, U: FeatureUnit, Main ctx=full_ctx, source=full_source, env=env, - _semaphore=semaphore, + _agent_semaphore=semaphore, + _cpu_semaphore=cpu_semaphore, _handler_factory=task_handler ) return await run_pipeline( @@ -293,6 +401,9 @@ async def cont[FormT: BackendResult, A: ArtifactIdentifier, U: FeatureUnit, Main interactive=args.interactive, max_bug_rounds=args.max_bug_rounds, threat_model=threat_model, + extra_context=extra_context, + budget=budget, + time_budget_s=args.time_budget, ecosystem=ecosystem, ) diff --git a/composer/pipeline/core.py b/composer/pipeline/core.py index 8ef01e48..144b6d2e 100644 --- a/composer/pipeline/core.py +++ b/composer/pipeline/core.py @@ -3,12 +3,22 @@ Phase chain — each link is immutable and its existence proves the prior phase ran, so ordering is a constructor dependency rather than a call-order convention; there is no half-initialized state: - Backend ──prepare_system──▶ PreparedSystem ──prepare_formalization──▶ Formalizer - (config, source) (.main: structure) (formalize / persist / report) - -A backend whose units all build on one *shared* artifact inserts a link: ``prepare_formalization`` returns -a :class:`StagedFormalizer`, and its ``begin`` — handed every unit's properties — is what produces the -:class:`Formalizer`. + Backend ──preflight──▶ Pre ──prepare_system──▶ PreparedSystem ──prepare_formalization──▶ Formalizer + (config, source) (built (.main: structure) (formalize / + workspace) persist / report) + +Two links are *overlapped* with the LLM steps they don't depend on, since both are usually builds: +``preflight`` runs alongside system analysis, and ``prepare_formalization`` alongside property +extraction. ``preflight`` is additionally a *gate*: it and the analysis run in a task group, so a +failure on either side cancels the other rather than letting it spend on a run that can no longer +complete — a broken toolchain does not wait out the analysis agent, and a failed analysis does not +wait out the workspace build. The second pair is simply awaited in turn. + +A backend whose units all build on one *shared* artifact inserts a link rather than a call-order +convention: ``prepare_formalization`` returns a :class:`StagedFormalizer`, and its ``begin`` — handed +every unit's properties — is what produces the :class:`Formalizer`. Same rule as the rest of the +chain, so it needs no new rule: the artifact is a constructor argument to the only object that uses +it, and no formalizer ever exists without it. The driver owns the genuinely-shared steps: system analysis, per-component property extraction, the result-type-keyed cache, and (since the report is backend-agnostic) building + persisting the @@ -19,48 +29,128 @@ import asyncio import enum +import functools import logging from dataclasses import dataclass -from typing import Protocol, Any, cast -from collections.abc import Sequence +from typing import ( + Protocol, Any, ClassVar, Concatenate, cast, Awaitable, Sequence, Callable, ContextManager, overload +) from abc import ABC, abstractmethod +from contextlib import nullcontext from composer.io.multi_job import TaskInfo from composer.spec.artifacts import ArtifactStore from composer.spec.context import ( - WorkflowContext, CacheKey, Properties, ComponentGroup, SourceCode + WorkflowContext, ComponentGroup ) from composer.spec.system_model import ( - BaseApplication, ContractComponentInstance, FeatureUnit + BaseApplication, FeatureUnit ) -from composer.spec.types import PropertyFormulation, ArtifactIdentifier +from composer.spec.util import combine_digests +from composer.spec.types import PropertyFormulation, ArtifactIdentifier, VerificationArtifact from composer.spec.system_analysis import run_component_analysis from composer.spec.prop_inference import ( run_property_inference, AnyPropertyGenerationInput, CacheablePropertyGenerationInput, ) from composer.llm.types import CacheLevel -from composer.spec.util import string_hash from composer.input.files import Document from composer.spec.source.report.build import build_report -from composer.spec.source.report.collect import ReportComponentInput, Verdict, EvidenceFetcher +from composer.spec.source.report.collect import ReportComponentInput, Verdict, EvidenceFetcher, Formalized from composer.spec.source.report.schema import ( - AutoProverReport, RuleName, ReportBackend, SourceEditRecord, + AutoProverReport, RuleName, ReportBackend, SourceEditRecord, VerificationArtifactRecord, ) from composer.spec.source.report import build as report_build from composer.spec.source.task_ids import SYSTEM_ANALYSIS_TASK_ID, REPORT_TASK_ID from composer.pipeline.ecosystem import Ecosystem +from .keys import ( + COMPONENT_KEY, FINAL_PROPERTIES_KEY, FORMALIZATION_KEY, PLUGIN_ARTIFACTS_KEY, + POST_PROPERTY_KEY, PRE_PROPERTY_KEY, PROPERTIES_KEY, SYSTEM_ANALYSIS_KEY, + PLUGIN_FORMALIZATION_KEY +) +from composer.diagnostics.budget import total_budget, named_budget_or_nop, time_budget + from .ptypes import ( - BackendJob, BackendResult, ComponentOutcome, CorePhases, CorePipelineResult, Delivered, GaveUp, - PipelineRun, SystemAnalysisSpec + DEFAULT_MAX_CPU_TASKS, + BackendJob, BackendResult, ComponentOutcome, CorePhases, CorePipelineResult, + Curtailed, Delivered, RunBudget, + FinalProperties, GaveUp, PersistedPluginArtifact, PipelineRun, PluginArtifact, + RegisteredArtifacts, SystemAnalysisSpec +) +from .plugin_api import ( + AnyBackendTools, ArtifactRegistrar, FormalizationTool, PipelinePlugin, + PluginToolContext, ProvidedTools, ) -from .plugin_api import PrePropertyInference, PostPropertyInference from .plugins import load_plugins, PluginManager, PluginPhaseManager, PluginPhaseRunner -COMMON_SYSTEM_CACHE_KEY = "system-analysis" - _log = logging.getLogger(__name__) + +@dataclass +class ToolExtension[TP: PipelinePlugin[Any], U: FeatureUnit, **P]: + """A backend's tool-extension point, handed to the :class:`ToolBinder` at + dispatch time: the provider class whose derivers contribute (``provider``) and + the selection of their hook method (``project`` — canonically + ``lambda p: p._tools``, threading the dispatch leg itself, not a + wrapper). ``P`` captures the hook's backend-specific tail (its ``st`` / + ``state_reader``), which the binder forwards from the dispatch call. The bound + ``TP <: PipelinePlugin[U]`` is inexpressible, so unit agreement rides + ``project``'s signature statically — and, at runtime, the provider classes + deriving ``PipelinePlugin[U]`` plus the load-time scope check.""" + provider: type[TP] + project: Callable[ + [TP], + Callable[ + Concatenate[U, Sequence[PropertyFormulation], PluginToolContext[FormalizationTool], P], + Awaitable[ProvidedTools | None], + ], + ] + +@dataclass +class InjectingToolExtension[ + TP: PipelinePlugin[Any], + U: FeatureUnit, + **P, + Ext +]: + """A :class:`ToolExtension` whose hook additionally receives a backend-supplied + callable. The backend passes the binder a ``Callable[Concatenate[str, I], R]``; + the binder curries in the asking plugin's id, so the hook sees a + ``Callable[I, R]`` that already knows who it is serving — attribution is + stamped by the driver, never self-reported (the registrar in + :class:`_ArtifactCollector` works the same way).""" + provider: type[TP] + project: Callable[ + [TP], + Callable[ + Concatenate[U, Sequence[PropertyFormulation], PluginToolContext[FormalizationTool], Ext, P], + Awaitable[ProvidedTools | None] + ] + ] + + + +class ToolBinder[U: FeatureUnit](Protocol): + """What ``formalize`` receives — the core ↔ backend tool seam, naming no + backend. The backend calls it exactly once, with its own + :class:`ToolExtension` and the extension's tail arguments (its authoring + graph's state type and staged-state reader); the binder applies every + contributing plugin's hook — always including :class:`AnyBackendTools` + derivers — and returns the yielded bundles. Empty when no plugin contributes, + so backends need no special-casing.""" + + @overload + async def __call__[TP: PipelinePlugin[Any], **P]( + self, ext: ToolExtension[TP, U, P], *args: P.args, **kwargs: P.kwargs + ) -> Sequence[ProvidedTools]: + ... + + @overload + async def __call__[TP: PipelinePlugin[Any], **P, **I, R]( + self, ext: InjectingToolExtension[TP, U, P, Callable[I, R]], inj: Callable[Concatenate[str, I], R], /, *args: P.args, **kwargs: P.kwargs + ) -> Sequence[ProvidedTools]: + ... + @dataclass class Formalizer[FormT: BackendResult, U: FeatureUnit](ABC): """Immutable, fully constructed by whatever produced it — ``prepare_formalization``, or @@ -74,6 +164,12 @@ class Formalizer[FormT: BackendResult, U: FeatureUnit](ABC): formalized_type: type[FormT] backend_tag: ReportBackend + #: The provider class whose derivers contribute tools to this backend — the + #: driver matches it for both the dispatch gate and the formalization cache + #: key. None (the default) for a backend with no tool extension; + #: :class:`AnyBackendTools` derivers count regardless. + tool_provider_type: ClassVar[type[PipelinePlugin[Any]] | None] = None + @abstractmethod async def formalize( self, @@ -81,8 +177,13 @@ async def formalize( feat: U, props: list[PropertyFormulation], ctx: WorkflowContext[FormT], - run: PipelineRun - ) -> FormT | GaveUp: ... + run: PipelineRun, + extra_tools: ToolBinder[U] + ) -> FormT | Curtailed[FormT] | GaveUp: + """``extra_tools`` is the run's tool binder: call it exactly once with this + backend's :class:`ToolExtension` and the extension's tail (the authoring + graph's state type and staged-state reader).""" + ... def extra_report_inputs(self) -> list[ReportComponentInput[FormT]]: """Synthetic report inputs beyond the per-component outcomes — the prover folds in its @@ -97,9 +198,10 @@ async def source_edits( return [] @abstractmethod - async def fetch_verdicts(self, inp: ReportComponentInput[FormT]) -> dict[RuleName, Verdict]: - """Per-unit outcomes. Prover: query ProverOutputUtility via inp.formalized.run_link - off-thread. Foundry: read straight off inp.formalized.result.""" + async def fetch_verdicts(self, formalized: Formalized[FormT]) -> dict[RuleName, Verdict]: + """Per-unit outcomes for one delivered result. Prover: query ProverOutputUtility via + ``formalized.run_link`` off-thread. Foundry: read straight off ``formalized.result``. + Never called for gave-up or budget-curtailed components.""" ... def findings_evidence(self) -> EvidenceFetcher | None: @@ -159,7 +261,7 @@ async def prepare_formalization( ... -class PipelineBackend[P: enum.Enum, FormT: BackendResult, H, A: ArtifactIdentifier, U: FeatureUnit, Main, App: BaseApplication](Protocol): +class PipelineBackend[P: enum.Enum, FormT: BackendResult, H, A: ArtifactIdentifier, U: FeatureUnit, Main, App: BaseApplication, Pre](Protocol): @property def backend_guidance(self) -> str: ... @@ -172,40 +274,36 @@ def core_phases(self) -> CorePhases[P]: ... @property def artifact_store(self) -> ArtifactStore[A, FormT]: ... + + async def preflight(self, run: PipelineRun[P, H]) -> Pre: + """Whatever the backend can do before it knows anything about the program — run + *concurrently with system analysis*, and awaited before :meth:`prepare_system`. + + This is where a backend that must build something puts the build. Crucible prepares its + harness crate, compiles the program to sBPF, and gates a wheel-authored skeleton harness + through the real toolchain; none of that reads the analyzed model, so serializing it behind + analysis buys nothing, while running it alongside means a broken dependency graph or an + unbuildable program surfaces before the run has spent meaningfully on the model. + + ``Pre`` is opaque to the driver — it only carries the result to ``prepare_system``, so a + backend can hand its own prep forward as immutable state rather than stashing it on itself. + ``None`` for a backend with nothing to do ahead of time.""" + ... + async def prepare_system( self, analyzed: App, - run: PipelineRun[P, H] + run: PipelineRun[P, H], + preflight: Pre, ) -> PreparedSystem[FormT, U, Main]: ... def to_artifact_id(self, c: U) -> A: ... -# ---- shared helpers (the de-duplicated cache keys + batch) ------------------- -def PROPERTIES_KEY(nm: str): - return CacheKey[None, Properties](nm) - - +# ---- the per-unit batch ------------------------------------------------------- @dataclass class _Batch[U: FeatureUnit](BackendJob[U]): feat_ctx: WorkflowContext[ComponentGroup] -def _component_digest(c: FeatureUnit) -> str: - # ``cache_material`` is the ecosystem-agnostic view of what identifies a unit; EVM's - # implementation reproduces the previous inline key (app JSON | ind | contract ind) exactly. - return string_hash(c.cache_material()) - -def _component_cache_key(c: FeatureUnit, plugin_digest: str | None) -> CacheKey[Properties, ComponentGroup]: - raw_digest = _component_digest(c) - if plugin_digest is not None: - raw_digest += f"-{plugin_digest}" - return CacheKey(raw_digest) - - -def _batch_cache_key[FormT: BackendResult]( - props: list[PropertyFormulation], -) -> CacheKey[ComponentGroup, FormT]: # pyright: ignore[reportInvalidTypeVarUse] - return CacheKey(string_hash("|".join(p.model_dump_json() for p in props))) - def extract_task_id(idx: int) -> str: return f"extract-{idx}" @@ -214,31 +312,161 @@ def extract_task_id(idx: int) -> str: def formalize_task_id(idx: int) -> str: return f"formalize-{idx}" -async def run_pipeline[P: enum.Enum, FormT: BackendResult, H, A: ArtifactIdentifier, U: FeatureUnit, Main, App: BaseApplication]( - backend: PipelineBackend[P, FormT, H, A, U, Main, App], + +# ---- plugin tool contribution ------------------------------------------------- + +class _ArtifactCollector: + """Per-batch sink for the verification artifacts plugin tools register at + dispatch time. ``for_plugin`` stamps the attribution, so a plugin never + handles its own id; the driver snapshots the collected set into the + formalization namespace (cache replays skip the tools) and persists it via + the run's artifact store.""" + + def __init__(self) -> None: + self.items: list[PluginArtifact] = [] + + def for_plugin(self, plugin_id: str) -> ArtifactRegistrar: + collector = self + + class _Registrar: + def register(self, artifact: VerificationArtifact) -> None: + collector.items.append( + PluginArtifact(plugin=plugin_id, artifact=artifact) + ) + + return _Registrar() + + +def _contributes_tools( + plugin: PipelinePlugin[Any], provider_type: type[PipelinePlugin[Any]] | None +) -> bool: + """Whether ``plugin`` counts for the running backend's tool dispatch — and so + for its ``FORMALIZATION_KEY``: a deriver of the formalizer's declared provider + class, or of the always-projected :class:`AnyBackendTools`. One predicate for + the gate and the key, so what the binder can reach and what the key says + cannot drift.""" + if isinstance(plugin, AnyBackendTools): + return True + return provider_type is not None and isinstance(plugin, provider_type) + + +@dataclass +class _Binder[U: FeatureUnit]: + """Core's :class:`ToolBinder` over the batch's contributing plugins, each + pre-paired with its id and plugin context. Plugins fire in the deterministic + plugin-id order the gate collected them (injection order is prompt content); + per plugin, the extension's hook fires before the any-backend hook. Plugin + hook code runs only here — at the backend's dispatch — never as a top-level + tool-setup task. On an :class:`InjectingToolExtension` dispatch, the + backend's injectable gets the asking plugin's id curried in before the hook + ever sees it.""" + _plugins: Sequence[tuple[str, PipelinePlugin[U], PluginToolContext[FormalizationTool]]] + _comp: U + _props: list[PropertyFormulation] + + @overload + async def __call__[TP: PipelinePlugin[Any], **P]( + self, ext: ToolExtension[TP, U, P], *args: P.args, **kwargs: P.kwargs + ) -> Sequence[ProvidedTools]: + ... + + @overload + async def __call__[TP: PipelinePlugin[Any], **P, **I, R]( + self, ext: InjectingToolExtension[TP, U, P, Callable[I, R]], inj: Callable[Concatenate[str, I], R], /, *args: P.args, **kwargs: P.kwargs + ) -> Sequence[ProvidedTools]: + ... + + async def __call__( #type: ignore[trust me bro] + self, + ext: ToolExtension[Any, U, ...] | InjectingToolExtension[Any, U, ..., Any], + *args: Any, + **kwargs: Any, + ) -> Sequence[ProvidedTools]: + got: list[ProvidedTools] = [] + for plugin_id, p, ctxt in self._plugins: + if isinstance(p, ext.provider): + if isinstance(ext, InjectingToolExtension): + # args[0] is the backend's injectable (positional-only in + # the overload); the hook receives it with this plugin's + # id already bound. + t = await ext.project(p)( + self._comp, self._props, ctxt, + functools.partial(args[0], plugin_id), + *args[1:], **kwargs, + ) + else: + t = await ext.project(p)(self._comp, self._props, ctxt, *args, **kwargs) + if t is not None: + got.append(t) + if isinstance(p, AnyBackendTools): + t = await p.backend_tools(self._comp, self._props, ctxt) + if t is not None: + got.append(t) + return got + + + +def _budget_context(budget: RunBudget | None) -> ContextManager[None]: + if budget is not None: + return total_budget(budget.total, cast(dict[str, float], budget.caps)) + else: + return nullcontext() + +def _time_context(time_budget_s: float | None) -> ContextManager[None]: + if time_budget_s is not None: + return time_budget(time_budget_s) + else: + return nullcontext() + +async def run_pipeline[P: enum.Enum, FormT: BackendResult, H, A: ArtifactIdentifier, U: FeatureUnit, Main, App: BaseApplication, Pre]( + backend: PipelineBackend[P, FormT, H, A, U, Main, App, Pre], run: PipelineRun[P, H], *, interactive: bool = False, threat_model: Document | None = None, + extra_context: Sequence[Document] = (), max_bug_rounds: int = 3, ecosystem: Ecosystem[App, Main, U], + budget: RunBudget | None = None, + time_budget_s : float | None = None +) -> CorePipelineResult[FormT]: + with ( + _budget_context(budget), + _time_context(time_budget_s) + ): + return await _run_pipeline_inner( + backend, run, interactive=interactive, + max_bug_rounds=max_bug_rounds, threat_model=threat_model, + extra_context=extra_context, ecosystem=ecosystem + ) + +async def _run_pipeline_inner[P: enum.Enum, FormT: BackendResult, H, A: ArtifactIdentifier, U: FeatureUnit, Main, App: BaseApplication, Pre]( + backend: PipelineBackend[P, FormT, H, A, U, Main, App, Pre], + run: PipelineRun[P, H], + *, + interactive: bool, + threat_model: Document | None, + extra_context: Sequence[Document], + max_bug_rounds: int, + ecosystem: Ecosystem[App, Main, U], ) -> CorePipelineResult[FormT]: # Only the plugins whose hooks accept this ecosystem's unit are loaded (and only those pay # their ``initialize`` cost); the driver below can hand them its units unconditionally. async with load_plugins(run, ecosystem.unit_type) as plugins: return await run_pipeline_inner( backend, run, plugins, interactive=interactive, threat_model=threat_model, - max_bug_rounds=max_bug_rounds, ecosystem=ecosystem, + extra_context=extra_context, max_bug_rounds=max_bug_rounds, ecosystem=ecosystem, ) # ---- the driver -------------------------------------------------------------- -async def run_pipeline_inner[P: enum.Enum, FormT: BackendResult, H, A: ArtifactIdentifier, U: FeatureUnit, Main, App: BaseApplication]( - backend: PipelineBackend[P, FormT, H, A, U, Main, App], +async def run_pipeline_inner[P: enum.Enum, FormT: BackendResult, H, A: ArtifactIdentifier, U: FeatureUnit, Main, App: BaseApplication, Pre]( + backend: PipelineBackend[P, FormT, H, A, U, Main, App, Pre], run: PipelineRun[P, H], plugin_manager: PluginManager[P, U], *, interactive: bool = False, threat_model: Document | None = None, + extra_context: Sequence[Document] = (), max_bug_rounds: int = 3, ecosystem: Ecosystem[App, Main, U], ) -> CorePipelineResult[FormT]: @@ -249,29 +477,57 @@ async def run_pipeline_inner[P: enum.Enum, FormT: BackendResult, H, A: ArtifactI f"ecosystem {ecosystem.name!r} has no greenfield prompts; got sort='greenfield'" ) - # 1. System analysis (shared primitive; the ecosystem supplies the analyzed model type, - # prompts, validation, and front-matter — EVM reproduces prior behavior exactly). - analyzed = await run.runner( - TaskInfo(SYSTEM_ANALYSIS_TASK_ID, "System Analysis", phases["analysis"]), - lambda: run_component_analysis( - ty=ecosystem.system_model, child_ctxt=run.ctx.child(CacheKey(spec.analysis_key)), - input=source, env=run.env, - extra_input=[*ecosystem.analysis_extra_input(source), *spec.extra_input], - expected_main_id=source.contract_name, - system_template=ecosystem.analysis_prompts.system, - initial_template=ecosystem.analysis_prompts.initial, - validate=ecosystem.validate_analysis, - ), - ) + # 1. Backend preflight ∥ system analysis. The preflight (Crucible: build the program and gate a + # skeleton harness through the real toolchain) reads nothing analysis produces, so it starts + # at t=0. Neither side outlives the other's failure: whichever breaks first, the group cancels + # the one still running rather than let it spend — money on the analysis agent, minutes on the + # workspace build — on a run that can no longer complete. The analysis itself is the shared + # primitive; the ecosystem supplies the analyzed model type, prompts, validation, front-matter. + # The budget scope is entered inside the coroutine (not around create_task) so the + # cost-center binding lives in the spawned task's own context. + async def _run_analysis(): + with named_budget_or_nop("system_analysis"): + return await run.runner( + TaskInfo(SYSTEM_ANALYSIS_TASK_ID, "System Analysis", phases["analysis"]), + lambda: run_component_analysis( + ty=ecosystem.system_model, + child_ctxt=run.ctx.child(SYSTEM_ANALYSIS_KEY(ecosystem.system_model, spec.analysis_key)), + input=source, env=run.env, + extra_input=[*ecosystem.analysis_extra_input(source), *spec.extra_input], + expected_main_id=source.contract_name, + system_template=ecosystem.analysis_prompts.system, + initial_template=ecosystem.analysis_prompts.initial, + validate=ecosystem.validate_analysis, + ), + ) + + try: + async with asyncio.TaskGroup() as overlap: + preflight_task = overlap.create_task(backend.preflight(run)) + analysis_task = overlap.create_task(_run_analysis()) + except BaseExceptionGroup as eg: + # Callers expect the failure itself, not a wrapper, so unwrap the usual case: one side + # failed and the other was cancelled, and a cancelled task adds nothing to the group. + # Both failing at once is the only case with two real errors; keep the group there. + if len(eg.exceptions) == 1: + raise eg.exceptions[0] from None + raise + preflight, analyzed = preflight_task.result(), analysis_task.result() if analyzed is None: raise ValueError("System analysis produced no result.") # 2. Backend transform + main-contract location (prover: harness lift; foundry: identity). - prepared = await backend.prepare_system(analyzed, run) + with named_budget_or_nop("system_preparation"): + prepared = await backend.prepare_system(analyzed, run, preflight) # 3. Pre-formalization setup runs CONCURRENTLY with extraction (neither needs the other) — # this preserves the prover's autosetup ∥ bug-analysis overlap, generically. - staged_task = asyncio.create_task(prepared.prepare_formalization(run)) + # The budget scope is entered inside the coroutine (not around create_task) so the + # cost-center binding lives in the spawned task's own context. + async def _prepare_formalization() -> Formalizer[FormT, U] | StagedFormalizer[FormT, U]: + with named_budget_or_nop("formalization_preparation"): + return await prepared.prepare_formalization(run) + staged_task = asyncio.create_task(_prepare_formalization()) batches: list[_Batch[U]] = await _extract_all( backend.analysis_spec.properties_key, @@ -281,11 +537,11 @@ async def run_pipeline_inner[P: enum.Enum, FormT: BackendResult, H, A: ArtifactI phases["extraction"], interactive, threat_model, + extra_context, max_bug_rounds, ecosystem, plugin_manager.bind_phase( phases.get("extraction_plugin") or phases["extraction"], - "Property Extraction" ) ) staged = await staged_task @@ -295,40 +551,119 @@ async def run_pipeline_inner[P: enum.Enum, FormT: BackendResult, H, A: ArtifactI # 4. A backend whose units share an artifact handed back a ``StagedFormalizer`` instead of a # formalizer: the artifact is authored HERE — once, from every unit's properties — and the # formalizer it yields is the only one that exists (see :class:`StagedFormalizer`). - formalizer = ( - await staged.begin(batches, run) if isinstance(staged, StagedFormalizer) else staged - ) + with named_budget_or_nop("formalization_preparation"): + formalizer = ( + await staged.begin(batches, run) if isinstance(staged, StagedFormalizer) else staged + ) # 5. Per-component formalization. Caching is core-owned, keyed by the backend's result type. async def _run(batch: _Batch[U]) -> ComponentOutcome[FormT, U]: result_key = backend.to_artifact_id(batch.feat) backend.artifact_store.write_properties(result_key, batch.props) + + # Deriving the backend's declared provider class gates the dispatch AND + # keys the cache: whether a hook would actually yield tools is only + # knowable inside ``formalize`` (dispatch needs backend state that + # doesn't exist yet), so both run off what the plugin derives. A plugin + # outside this backend's gate is invisible to its binder, and so must + # not perturb its key either. No task machinery: tool hooks are + # tool-BINDING hooks, so they get the runner-less context and cannot + # run top-level tasks. + contributing_plugins : list[str] = [] + bound_plugins : list[tuple[str, PipelinePlugin[U], PluginToolContext[FormalizationTool]]] = [] + collected = _ArtifactCollector() + + for plugin_id, plugin in plugin_manager.sorted_plugins(): + if not _contributes_tools(plugin, formalizer.tool_provider_type): + continue + k = batch.feat_ctx.child(PLUGIN_FORMALIZATION_KEY(plugin_id)) + bound_plugins.append((plugin_id, plugin, plugin_manager.tool_context( + plugin, k, collected.for_plugin(plugin_id), + ))) + contributing_plugins.append(plugin_id) + + # Record the formalization edge's exact derivation inputs — the final + # (post-plugin) batch and the gated plugin ids — as a first-class cache + # entry, keyed like the bug analysis (the component namespace is shared + # across runs). Offline walkers reconstruct the edge from this instead + # of sniffing the store. + await batch.feat_ctx.child( + FINAL_PROPERTIES_KEY( + threat_model.to_digest() if threat_model is not None else None, + interactive, + combine_digests([d.to_digest() for d in extra_context]), + ) + ).cache_put(FinalProperties(items=batch.props, tool_plugins=contributing_plugins)) + child : WorkflowContext[FormT] = await batch.feat_ctx.child( - _batch_cache_key(batch.props), + FORMALIZATION_KEY(formalizer.formalized_type, batch.props, contributing_plugins), {"properties": [p.model_dump() for p in batch.props]}, ) + artifacts_ctx = child.child(PLUGIN_ARTIFACTS_KEY) cached_result: FormT | None = await child.cache_get(formalizer.formalized_type) - result : FormT | GaveUp + result : FormT | Curtailed[FormT] | GaveUp if cached_result is None: label = f"{batch.feat.display_name} ({len(batch.props)} properties)" - result : FormT | GaveUp = await run.runner( - TaskInfo( - formalize_task_id(batch.feat.unit_index), - label, - phases["formalization"] - ), - lambda: formalizer.formalize(label, batch.feat, batch.props, child, run), - ) + with named_budget_or_nop("formalization"): + result : FormT | GaveUp | Curtailed[FormT] = await run.runner( + TaskInfo( + formalize_task_id(batch.feat.unit_index), + label, + phases["formalization"] + ), + lambda: formalizer.formalize( + label, batch.feat, batch.props, child, run, + _Binder(bound_plugins, batch.feat, batch.props) + ), + ) + # Snapshot what the tools registered: a cache replay of this + # formalization never runs the tools, so the artifacts must ride + # the cache alongside the result they accompany. if not isinstance(result, GaveUp): - await child.cache_put(result) + await artifacts_ctx.cache_put(RegisteredArtifacts(items=collected.items)) + # Envelope before result: a crash between the two puts replays + # as a miss (re-run, envelope rewritten) rather than a hit with + # silently absent artifacts. A curtailed partial is never the + # component's cached result — a later run redoes the + # formalization under a fresh budget. + if not isinstance(result, Curtailed): + await child.cache_put(result) else: result = cached_result - - outcome: Delivered[FormT] | GaveUp = ( - result if isinstance(result, GaveUp) - else Delivered(result, backend.artifact_store.write_artifact(result_key, result)) - ) - return ComponentOutcome(batch.feat, batch.props, outcome) + restored = await artifacts_ctx.cache_get(RegisteredArtifacts) + collected.items = restored.items if restored is not None else [] + + outcome: Delivered[FormT] | Curtailed[Delivered[FormT]] | GaveUp + if isinstance(result, GaveUp): + outcome = result + elif isinstance(result, Curtailed): + # Persist the partial for inspection under a quarantined name — never as the + # component's deliverable. + outcome = Curtailed( + Delivered( + result.partial, + backend.artifact_store.write_quarantined(result_key, result.partial), + ) if result.partial is not None else None, + result.detail, + ) + else: + outcome = Delivered(result, backend.artifact_store.write_artifact(result_key, result)) + + # Persist registered artifacts on every path (fresh run or cache + # replay), like ``write_artifact`` above — deliverables are rebuilt + # from cache content, never assumed to survive on disk. A gave-up + # component keeps whatever its tools registered before the surrender. + persisted = [ + PersistedPluginArtifact( + plugin=pa.plugin, + artifact=pa.artifact, + path=backend.artifact_store.write_plugin_artifact( + result_key, pa.plugin, pa.artifact + ), + ) + for pa in collected.items + ] + return ComponentOutcome(batch.feat, batch.props, outcome, persisted) settled = await asyncio.gather(*[_run(b) for b in batches], return_exceptions=True) outcomes = [o if isinstance(o, ComponentOutcome) @@ -344,10 +679,22 @@ async def _run(batch: _Batch[U]) -> ComponentOutcome[FormT, U]: ReportComponentInput( name=o.feat.display_name, props=o.props, - formalized=o.result if isinstance(o.result, Delivered) else None, + formalized=o.result if isinstance(o.result, (Delivered, Curtailed)) else None, ) for o in outcomes ] + formalizer.extra_report_inputs() + artifact_records = [ + VerificationArtifactRecord( + component=o.feat.display_name, + plugin=pa.plugin, + name=pa.artifact.name, + kind=pa.artifact.kind, + description=pa.artifact.description, + path=str(pa.path), + ) + for o in outcomes + for pa in o.artifacts + ] findings_evidence = formalizer.findings_evidence() try: async def _report() -> AutoProverReport: @@ -355,6 +702,7 @@ async def _report() -> AutoProverReport: contract_name=source.contract_name, backend=formalizer.backend_tag, components=inputs, llm=run.env.llm_lite(), fetch_verdicts=formalizer.fetch_verdicts, source_edits=await formalizer.source_edits(outcomes, run), + verification_artifacts=artifact_records, # Findings only when the backend supplies evidence — skip the heavy model otherwise. findings_llm=run.env.llm_heavy() if findings_evidence else None, fetch_evidence=findings_evidence, @@ -372,21 +720,11 @@ async def _report() -> AutoProverReport: return _tally(outcomes) -def _pre_property_cache_key(feat: FeatureUnit, plugin: str) -> CacheKey[Properties, PrePropertyInference]: - key = f"{_component_digest(feat)}-{string_hash(plugin)}-pre" - return CacheKey(key) - -def _post_property_cache_key(feat: FeatureUnit, plugin: str, curr_props: list[PropertyFormulation]) -> CacheKey[Properties, PostPropertyInference]: - props = string_hash("|".join( - p.model_dump_json() for p in curr_props - )) - key = f"{_component_digest(feat)}-{string_hash(plugin)}-{props}" - return CacheKey(key) - async def _extract_all[P: enum.Enum, H, Main, U: FeatureUnit]( prop_key: str, main: Main, backend_guidance: str, run: PipelineRun[P, H], - phase: P, interactive: bool, threat_model: Document | None, max_rounds: int, + phase: P, interactive: bool, threat_model: Document | None, + extra_context: Sequence[Document], max_rounds: int, # ``App`` stays ``Any`` here: this helper never touches the analyzed-model axis, only # ``Main``/``U`` (matching the caller's), so there's nothing to tie it to. ecosystem: Ecosystem[Any, Main, U], @@ -397,7 +735,7 @@ async def _extract_all[P: enum.Enum, H, Main, U: FeatureUnit]( async def _pre_plugin_inputs(feat: U) -> list[AnyPropertyGenerationInput]: async def run_one(runner: PluginPhaseRunner[P, U]) -> AnyPropertyGenerationInput | None: ctxt = await prop_ctx.child( - _pre_property_cache_key(feat, runner.plugin_id), {"plugin-name": runner.plugin_id} + PRE_PROPERTY_KEY(feat, runner.plugin_id), {"plugin-name": runner.plugin_id} ) return await runner.plugin.property_inference_input_hook( feat, runner.bind(str(feat.unit_index), ctxt) @@ -418,7 +756,7 @@ async def _post_plugin_props( sub_phase_id="post-inference", sub_phase_label="Property Post-Process", sorted_run=True ): ctxt = await prop_ctx.child( - _post_property_cache_key(feat, runner.plugin_id, accum), + POST_PROPERTY_KEY(feat, runner.plugin_id, accum), { "plugin-name": runner.plugin_id, "props": [p.model_dump() for p in accum], @@ -442,21 +780,22 @@ async def _one(feat: U) -> _Batch[U] | None: CacheablePropertyGenerationInput( "certora:system-doc", "generic", "always", lambda cache, doc=design_doc: [ - "For reference, the system document describing the entire application is as follows.", + "For reference, the system document describing the entire application is as follows.\n\n", doc.to_dict(CacheLevel.SHORT if cache else CacheLevel.NONE) ] ), ] feat_ctx = await prop_ctx.child( - _component_cache_key(feat, plugins.plugin_digest), + COMPONENT_KEY(feat, plugins.plugin_digest), {**feat.context_tag(), "plugins": plugins.plugin_manifest}, ) props = await run.runner( TaskInfo(extract_task_id(feat.unit_index), feat.display_name, phase), lambda conv: run_property_inference( feat_ctx, run.env, feat, refinement=conv if interactive else None, - threat_model=threat_model, max_rounds=max_rounds, backend_guidance=backend_guidance, + threat_model=threat_model, extra_context=extra_context, + max_rounds=max_rounds, backend_guidance=backend_guidance, extra_input=pre_input, system_template=ecosystem.property_prompts.system, render_initial=ecosystem.property_prompts.render_initial, @@ -468,7 +807,11 @@ async def _one(feat: U) -> _Batch[U] | None: accum = await _post_plugin_props(feat, props) return _Batch(feat, accum, feat_ctx) if accum else None - got = await asyncio.gather(*[_one(u) for u in ecosystem.units(main)]) + async def budgeted_task(u: U) -> _Batch | None: + with named_budget_or_nop("property_extraction"): + return await _one(u) + + got = await asyncio.gather(*[budgeted_task(u) for u in ecosystem.units(main)]) return [b for b in got if b is not None] @@ -481,6 +824,14 @@ def _tally[FormT: BackendResult, U: FeatureUnit]( failures.append(f"{o.feat.display_name}: {o.result}") elif isinstance(o.result, GaveUp): failures.append(f"{o.feat.display_name}: GAVE_UP: {o.result.reason}") + elif isinstance(o.result, Curtailed): + what = ( + f"unvalidated partial kept at {o.result.partial.deliverable}" + if o.result.partial is not None else "nothing published" + ) + failures.append( + f"{o.feat.display_name}: BUDGET: formalization cut short ({what})" + ) # The rollup is unit-agnostic; widen the concrete-unit outcomes to the protocol for storage. return CorePipelineResult( len(outcomes), sum(len(o.props) for o in outcomes), diff --git a/composer/pipeline/ecosystem.py b/composer/pipeline/ecosystem.py index ee6826d7..69e65f78 100644 --- a/composer/pipeline/ecosystem.py +++ b/composer/pipeline/ecosystem.py @@ -16,10 +16,10 @@ from dataclasses import dataclass from pathlib import PurePath -from typing import Any, Callable, Literal, Mapping, TypedDict +from typing import Any, Callable, Collection, Literal, Mapping, TypedDict from composer.spec.context import SourceCode -from composer.spec.code_explorer import CODE_EXPLORER_SYS_PROMPT +from composer.spec.code_explorer import CodeExplorerPromptParams from composer.spec.gen_types import TypedTemplate from composer.spec.prop_inference import ( InitialPromptRenderer, @@ -54,6 +54,15 @@ SolanaProgram, SolanaProgramInstance, ) +from composer.spec.soroban.model import ( + AuthorityInteraction as SorobanAuthorityInteraction, + SorobanApplication, + SorobanAuthority, + SorobanComponentInstance, + SorobanContract, + SorobanContractInstance, + StorageDurability, +) from composer.spec.util import fs_forbidden_read, slugify_filename LanguageTag = Literal["solidity", "rust"] @@ -90,17 +99,17 @@ class Language: """The language of the **code being analyzed** — a facet of the ecosystem, shared by every chain whose programs are written in it (e.g. the ``rust`` facet is shared by Solana and Soroban). It drives how the shared front half *reads* the target's source (fs-exclusion - rule, code-explorer prompt, failure modes).""" + rule, failure modes). The code-explorer system prompt lives on :class:`Ecosystem`: its + look-fors are chain-shaped (PDAs vs ``require_auth``), not language-shaped.""" name: LanguageTag #: What the agent's source tools withhold: either a predicate over the project-root-relative #: path (Solidity's ``fs_forbidden_read``, whose carve-outs don't fit a regex) or a plain #: exclusion pattern where one suffices. Both shapes are what ``GlobalExcludeArg`` accepts. default_forbidden_read: str | Callable[[PurePath], bool] - code_explorer_prompt: str - # The j2 partial with this language's vulnerability patterns (overflow, panics, …). Reserved + # The j2 fragment with this language's vulnerability patterns (overflow, panics, …). Reserved # for the prompt-fragment split; unused while prompts are still monolithic. - vulnerability_patterns_partial: str | None = None + vulnerability_patterns_fragment: str | None = None @dataclass(frozen=True) @@ -141,6 +150,10 @@ class Ecosystem[App: BaseApplication, Main, Unit: FeatureUnit]: unit_type: type[Unit] #: Domain-specific front-matter appended to the analysis input (was hardcoded in the driver). analysis_extra_input: Callable[[SourceCode], list[str | dict]] + #: System prompt for the code-explorer sub-agent. Composed of the shared explorer protocol + #: plus this chain's look-fors (``composer/templates/code_explorer/``). Rendered with + #: :class:`~composer.spec.code_explorer.CodeExplorerPromptParams`. + code_explorer_prompt: TypedTemplate[CodeExplorerPromptParams] #: Whether ``analysis_prompts``/``property_prompts`` have a ``sort == "greenfield"`` branch. #: Only EVM does (the natspec design-doc-to-Solidity path) — Solana's templates have no #: greenfield content, so ``run_pipeline`` asserts against this rather than silently rendering @@ -148,11 +161,14 @@ class Ecosystem[App: BaseApplication, Main, Unit: FeatureUnit]: supports_greenfield: bool -#: Names for the two concrete instantiations, so a consumer that is pinned to one chain can say so +#: Names for the concrete instantiations, so a consumer that is pinned to one chain can say so #: without respelling the triple (or erasing it to ``Any``). :class:`Ecosystems` declares the same #: pairing; these are what its fields are typed with. type EvmEcosystem = Ecosystem[SourceApplication, ContractInstance, ContractComponentInstance] type SolanaEcosystem = Ecosystem[SolanaApplication, SolanaProgramInstance, SolanaComponentInstance] +type SorobanEcosystem = Ecosystem[ + SorobanApplication, SorobanContractInstance, SorobanComponentInstance +] # --------------------------------------------------------------------------- @@ -193,9 +209,9 @@ def _evm_analysis_extra_input(source: SourceCode) -> list[str | dict]: # Adding Vyper support (a second EVM source language) would, at a very high level: # 1. Extend ``LanguageTag`` with ``"vyper"`` and add a ``VYPER`` ``Language`` facet here (its -# own ``forbidden_read``, code-explorer prompt, and — eventually — failure-modes partial). -# 2. Bind it to a Vyper-flavored EVM ``Ecosystem`` (its own analysis/property prompts) and -# route to it by detecting the target's source language at the entry point. +# own ``forbidden_read`` and — eventually — failure-modes fragment). +# 2. Bind it to a Vyper-flavored EVM ``Ecosystem`` (its own analysis/property/code-explorer +# prompts) and route to it by detecting the target's source language at the entry point. # 3. Loosen the analysis model's Solidity assumptions: contracts are keyed by # ``SolidityIdentifier`` / ``solidity_identifier`` throughout (see ``system_model`` and # ``main_instance``), which would need to widen to the language-neutral @@ -206,9 +222,11 @@ def _evm_analysis_extra_input(source: SourceCode) -> list[str | dict]: SOLIDITY = Language( name="solidity", default_forbidden_read=fs_forbidden_read, - code_explorer_prompt=CODE_EXPLORER_SYS_PROMPT, ) +#: Bound at top level so ``composer.meta.templates``'s AST scan can see it. +EVM_CODE_EXPLORER_TEMPLATE = TypedTemplate[CodeExplorerPromptParams]("code_explorer/solidity.j2") + EVM: EvmEcosystem = Ecosystem( name="evm", language=SOLIDITY, @@ -221,6 +239,7 @@ def _evm_analysis_extra_input(source: SourceCode) -> list[str | dict]: units=_evm_units, unit_type=ContractComponentInstance, analysis_extra_input=_evm_analysis_extra_input, + code_explorer_prompt=EVM_CODE_EXPLORER_TEMPLATE, ) @@ -233,24 +252,16 @@ def _evm_analysis_extra_input(source: SourceCode) -> list[str | dict]: #: ``fs_forbidden_read``, nothing needs carving back out of an excluded directory. RUST_FORBIDDEN_READ = r"(^target/.*)|(^\.git.*)|(^node_modules/.*)|(.*\.lock$)" # NOTE: the confined-build scratch dirs (``.sandbox_cargo`` / ``.sandbox_rustup`` / -# ``.sandbox_tmp`` and nested ``target/``) are also excluded, but that extension lives with the -# rust-framework layer that introduces confined Rust builds — no build runs in this front-half, so -# those dirs never exist here. - -RUST_CODE_EXPLORER_PROMPT = """\ -You are a code-exploration assistant analyzing Rust source for on-chain programs (e.g. Solana -/ Anchor). You have file tools (list_files, get_file, grep_files) to explore the project. -Answer the question concretely, citing the relevant items: instruction handlers, account -validation structs (e.g. Anchor `#[derive(Accounts)]`), account/state types, PDA seed -derivations, signer/owner checks, and cross-program invocations. Quote the exact Rust snippets -that establish or omit a check; do not speculate about code you have not read. -""" +# ``.sandbox_tmp`` and nested ``target/``) also have to be excluded — a build fills them with +# hundreds of MB the source tools' file-listing would pull into the model's context — but that +# extension lives with the *backend* that runs confined Rust builds inside the workdir. Nothing in +# the front half, and nothing in the Rust application framework itself, creates them: a Rust +# backend need not build a crate to validate the program, nor use the sandbox at all. RUST = Language( name="rust", default_forbidden_read=RUST_FORBIDDEN_READ, - code_explorer_prompt=RUST_CODE_EXPLORER_PROMPT, - vulnerability_patterns_partial="rust/_vulnerability_patterns.j2", + vulnerability_patterns_fragment="rust/vulnerability_patterns_fragment.j2", ) @@ -259,6 +270,25 @@ def _evm_analysis_extra_input(source: SourceCode) -> list[str | dict]: # --------------------------------------------------------------------------- +def _fmt_names(items: Collection[str]) -> str: + return ", ".join(sorted(items)) if items else "(none)" + + +def _analysis_feedback(errors: list[str], reference_lines: list[str]) -> str | None: + if not errors: + return None + reference = ( + "\n\nFor reference, the names you declared in your submission:\n" + "\n".join(reference_lines) + ) + if len(errors) == 1: + return errors[0] + reference + return ( + "Multiple validation errors; fix all before resubmitting:\n" + + "\n".join(f"- {e}" for e in errors) + + reference + ) + + def _validate_program_components(prog: SolanaProgram) -> list[str]: """One program's :class:`ProgramComponent` checks — the peer of the component half of EVM's ``validate_solidity_connectivity`` (``docs/ecosystem-abstraction.md`` §4, "Validation"). @@ -368,31 +398,14 @@ def _solana_validate(app: SolanaApplication, expected_main: SourceIdentifier | N f"Expected a program with identifier {expected_main!r}; declared programs: " f"{sorted(known_identifiers) or '(none)'}." ) - if not errors: - return None - - # The declared-names reference block (EVM peer): every error above is a name that failed to - # resolve, so the retry is far more likely to land if it can see the vocabulary it submitted. - def _fmt(items: set[str]) -> str: - return ", ".join(sorted(items)) if items else "(none)" reference_lines = [ - f"- Declared programs: {_fmt(set(known_components))}", - f"- Declared external authorities: {_fmt(known_authorities)}", + f"- Declared programs: {_fmt_names(set(known_components))}", + f"- Declared external authorities: {_fmt_names(known_authorities)}", ] for prog_name, comps in sorted(known_components.items()): - reference_lines.append(f"- Components of {prog_name}: {_fmt(comps)}") - reference = ( - "\n\nFor reference, the names you declared in your submission:\n" + "\n".join(reference_lines) - ) - - if len(errors) == 1: - return errors[0] + reference - return ( - "Multiple validation errors; fix all before resubmitting:\n" - + "\n".join(f"- {e}" for e in errors) - + reference - ) + reference_lines.append(f"- Components of {prog_name}: {_fmt_names(comps)}") + return _analysis_feedback(errors, reference_lines) def _solana_locate_main(app: SolanaApplication, source: SourceCode) -> SolanaProgramInstance: @@ -440,6 +453,7 @@ class SolanaPropertyPromptParams(TypedDict): SOLANA_ANALYSIS_INITIAL_TEMPLATE = TypedTemplate[AnalysisPromptParams]("solana/analysis_prompt.j2") SOLANA_PROPERTY_SYSTEM_TEMPLATE = TypedTemplate[PropertySystemPromptParams]("solana/property_system.j2") SOLANA_PROPERTY_INITIAL_TEMPLATE = TypedTemplate[SolanaPropertyPromptParams]("solana/property_prompt.j2") +SOLANA_CODE_EXPLORER_TEMPLATE = TypedTemplate[CodeExplorerPromptParams]("code_explorer/solana.j2") def _render_solana_property_prompt( @@ -473,6 +487,187 @@ def _render_solana_property_prompt( units=_solana_units, unit_type=SolanaComponentInstance, analysis_extra_input=_solana_analysis_extra_input, + code_explorer_prompt=SOLANA_CODE_EXPLORER_TEMPLATE, +) + + +# --------------------------------------------------------------------------- +# The Soroban chain (RUST ⊕ soroban) +# --------------------------------------------------------------------------- + + +def _validate_contract_components(contract: SorobanContract) -> list[str]: + errors: list[str] = [] + dup_keys: dict[str, StorageDurability] = {} + for entry in contract.storage_entries: + if entry.key in dup_keys: + errors.append( + f"Storage key {entry.key!r} is declared twice in {contract.name} (durabilities " + f"{dup_keys[entry.key]!r} and {entry.durability!r}). The three durabilities are " + f"separate key spaces, so give each entry a key that identifies it uniquely." + ) + else: + dup_keys[entry.key] = entry.durability + seen: set[str] = set() + slug_origin: dict[str, str] = {} + for comp in contract.components: + if comp.name in seen: + errors.append(f"Duplicate component names in {contract.name}: {comp.name}") + seen.add(comp.name) + slug = slugify_filename(comp.name) + if slug in slug_origin: + errors.append( + f"Components {slug_origin[slug]!r} and {comp.name!r} in {contract.name} both " + f"reduce to the filename slug {slug!r} (punctuation and symbols are normalized to " + f"underscores); give them names that differ in more than that." + ) + else: + slug_origin[slug] = comp.name + for fn_name in comp.functions: + if fn_name not in contract.functions_by_name: + errors.append( + f"Component {comp.name!r} of {contract.name} lists a function {fn_name!r} " + f"that {contract.name} does not declare." + ) + for key in comp.storage_keys: + if key not in contract.storage_by_key: + errors.append( + f"Component {comp.name!r} of {contract.name} lists a storage key {key!r} that " + f"is not among {contract.name}'s declared `storage_entries`." + ) + referenced = {n for comp in contract.components for n in comp.functions} + unassigned = [f.name for f in contract.functions if f.name not in referenced] + if unassigned: + errors.append( + f"Function(s) {', '.join(repr(n) for n in unassigned)} of {contract.name} belong to no " + f"component; every function must appear in at least one component's `functions` " + f"(a function may appear in more than one)." + ) + return errors + + +def _soroban_validate( + app: SorobanApplication, expected_main: SourceIdentifier | None +) -> str | None: + errors: list[str] = [] + known_identifiers: set[str] = set() + known_components: dict[str, set[str]] = {} + contracts = [c for c in app.components if isinstance(c, SorobanContract)] + known_authorities = {c.name for c in app.components if isinstance(c, SorobanAuthority)} + for contract in contracts: + if contract.contract_identifier in known_identifiers: + errors.append(f"Duplicate contract identifier: {contract.contract_identifier}") + known_identifiers.add(contract.contract_identifier) + if contract.name in known_components: + errors.append(f"Duplicate contract names: {contract.name}") + known_components.setdefault(contract.name, set()).update( + c.name for c in contract.components + ) + slug_origin: dict[str, str] = {} + for fn in contract.functions: + slug = slugify_filename(fn.name) + if slug in slug_origin: + errors.append( + f"Functions {slug_origin[slug]!r} and {fn.name!r} in {contract.name} " + f"reduce to the same filename slug {slug!r}; give them more-distinct names." + ) + slug_origin[slug] = fn.name + errors.extend(_validate_contract_components(contract)) + + for contract in contracts: + for comp in contract.components: + where = f"Component {comp.name} of {contract.name} interacts with" + for inter in comp.interactions: + if isinstance(inter, SorobanAuthorityInteraction): + if inter.authority not in known_authorities: + errors.append(f"{where} unknown external authority: {inter.authority}") + elif inter.contract not in known_components: + errors.append(f"{where} an unknown contract: {inter.contract}") + elif inter.component not in known_components[inter.contract]: + errors.append( + f"{where} unknown component {inter.component} of contract {inter.contract}" + ) + + if expected_main is not None and expected_main not in known_identifiers: + errors.append( + f"Expected a contract with identifier {expected_main!r}; declared contracts: " + f"{sorted(known_identifiers) or '(none)'}." + ) + + reference_lines = [ + f"- Declared contracts: {_fmt_names(set(known_components))}", + f"- Declared external authorities: {_fmt_names(known_authorities)}", + ] + for contract_name, comps in sorted(known_components.items()): + reference_lines.append(f"- Components of {contract_name}: {_fmt_names(comps)}") + return _analysis_feedback(errors, reference_lines) + + +def _soroban_locate_main(app: SorobanApplication, source: SourceCode) -> SorobanContractInstance: + for i, contract in enumerate(app.contracts): + if contract.contract_identifier == source.contract_name: + return SorobanContractInstance(i, app) + raise ValueError(f"main contract {source.contract_name!r} not found in analyzed application") + + +def _soroban_units(main: SorobanContractInstance) -> list[SorobanComponentInstance]: + return [ + SorobanComponentInstance(ind=i, _contract=main) + for i in range(len(main.contract.components)) + ] + + +def _soroban_analysis_extra_input(source: SourceCode) -> list[str | dict]: + return [ + f"The main contract of this application has been explicitly identified as " + f"{source.contract_name} at relative path {source.relative_path}. " + "Your output MUST contain a contract whose contract_identifier is this exact identifier." + ] + + +@component_context +class SorobanPropertyPromptParams(TypedDict): + context: SorobanComponentInstance + sort: Sort + prior_properties: list[_AgentRoundResult] + + +SOROBAN_ANALYSIS_SYSTEM_TEMPLATE = TypedTemplate[AnalysisPromptParams]("soroban/analysis_system.j2") +SOROBAN_ANALYSIS_INITIAL_TEMPLATE = TypedTemplate[AnalysisPromptParams]("soroban/analysis_prompt.j2") +SOROBAN_PROPERTY_SYSTEM_TEMPLATE = TypedTemplate[PropertySystemPromptParams]("soroban/property_system.j2") +SOROBAN_PROPERTY_INITIAL_TEMPLATE = TypedTemplate[SorobanPropertyPromptParams]("soroban/property_prompt.j2") +SOROBAN_CODE_EXPLORER_TEMPLATE = TypedTemplate[CodeExplorerPromptParams]("code_explorer/soroban.j2") + + +def _render_soroban_property_prompt( + context: SorobanComponentInstance, + sort: Sort, + prior_properties: list[_AgentRoundResult], +) -> str: + return SOROBAN_PROPERTY_INITIAL_TEMPLATE.bind({ + "context": context, + "sort": sort, + "prior_properties": prior_properties, + }).render_to(load_jinja_template) + + +SOROBAN: SorobanEcosystem = Ecosystem( + name="soroban", + language=RUST, + system_model=SorobanApplication, + analysis_prompts=PromptPair( + SOROBAN_ANALYSIS_SYSTEM_TEMPLATE, SOROBAN_ANALYSIS_INITIAL_TEMPLATE + ), + property_prompts=PropertyPrompts( + SOROBAN_PROPERTY_SYSTEM_TEMPLATE, _render_soroban_property_prompt + ), + validate_analysis=_soroban_validate, + locate_main=_soroban_locate_main, + supports_greenfield=False, + units=_soroban_units, + unit_type=SorobanComponentInstance, + analysis_extra_input=_soroban_analysis_extra_input, + code_explorer_prompt=SOROBAN_CODE_EXPLORER_TEMPLATE, ) @@ -484,6 +679,7 @@ class Ecosystems(TypedDict): evm: EvmEcosystem solana: SolanaEcosystem + soroban: SorobanEcosystem -ECOSYSTEMS: Ecosystems = {"evm": EVM, "solana": SOLANA} +ECOSYSTEMS: Ecosystems = {"evm": EVM, "solana": SOLANA, "soroban": SOROBAN} diff --git a/composer/pipeline/keys.py b/composer/pipeline/keys.py new file mode 100644 index 00000000..851b3186 --- /dev/null +++ b/composer/pipeline/keys.py @@ -0,0 +1,160 @@ +"""The Auto-Prove driver's cache tree, declared in one place. + +Every edge the backend-agnostic driver (``composer.pipeline.core``) +traverses below the run root, in tree order:: + + run root (``composer.pipeline.cli``: ``user_ns(cache_ns, root_cache_key(...))``) + ├── {analysis key} SYSTEM_ANALYSIS_KEY → ecosystem App model + └── {properties key} PROPERTIES_KEY → Properties + ├── {unit digest}[-{plugin digest}] COMPONENT_KEY → ComponentGroup + │ ├── bug_analysis[|refine][-tm-…][-xc-…] + │ │ BUG_ANALYSIS_KEY → _BugAnalysisCache + │ │ └── agent_bug_analysis AGENT_RESULT_KEY → _AgentResult + │ │ └── round-{i} AGENT_ROUND_KEY → _AgentRoundWithHistory + │ ├── final_props[|refine][-tm-…][-xc-…] + │ │ FINAL_PROPERTIES_KEY → FinalProperties + │ └── {props digest} FORMALIZATION_KEY → backend result (FormT) + │ └── plugin-artifacts PLUGIN_ARTIFACTS_KEY → RegisteredArtifacts + ├── {unit digest}-{plugin}-pre PRE_PROPERTY_KEY → PrePropertyInference + └── {unit digest}-{plugin}-{props} POST_PROPERTY_KEY → PostPropertyInference + +The extraction-layer families (bug analysis, agent rounds) are declared +in ``composer.spec.prop_inference`` beside their cache models and +re-exported here. The prover backend's sub-chain (config / harness / +autosetup / summaries / invariants / CVL generation) has its own +registry: ``composer.spec.source.keys``. +""" + +from typing import Any + +from composer.pipeline.ptypes import FinalProperties, RegisteredArtifacts +from composer.spec.context import CacheKey, ComponentGroup, Properties +from composer.spec.key_family import KeyFamily, PolyKeyFamily +from composer.spec.prop_inference import ( + AGENT_RESULT_KEY, AGENT_ROUND_KEY, BUG_ANALYSIS_KEY, +) +from composer.spec.system_model import FeatureUnit +from composer.spec.types import PropertyFormulation +from composer.spec.util import string_hash + +from .plugin_api import PostPropertyInference, PrePropertyInference, FormalizationTool + +__all__ = [ + "AGENT_RESULT_KEY", + "AGENT_ROUND_KEY", + "BUG_ANALYSIS_KEY", + "COMMON_SYSTEM_CACHE_KEY", + "COMPONENT_KEY", + "FINAL_PROPERTIES_KEY", + "FORMALIZATION_KEY", + "PLUGIN_ARTIFACTS_KEY", + "POST_PROPERTY_KEY", + "PRE_PROPERTY_KEY", + "PROPERTIES_KEY", + "SYSTEM_ANALYSIS_KEY", + "component_digest", +] + +#: The analysis-slot name both current backends declare in their +#: ``SystemAnalysisSpec``. +COMMON_SYSTEM_CACHE_KEY = "system-analysis" + + +def _slot_name(name: str) -> str: + return name + + +def component_digest(c: FeatureUnit) -> str: + """``cache_material`` is the ecosystem-agnostic view of what identifies a + unit; EVM's implementation reproduces the previous inline key + (app JSON | ind | contract ind) exactly.""" + return string_hash(c.cache_material()) + + +def _props_digest(props: list[PropertyFormulation]) -> str: + return string_hash("|".join(p.model_dump_json() for p in props)) + + +#: System analysis, keyed by the backend's declared slot name +#: (``SystemAnalysisSpec.analysis_key``). The child is the ecosystem's +#: analyzed model — pass ``ecosystem.system_model``. +SYSTEM_ANALYSIS_KEY = PolyKeyFamily(type(None), _slot_name) + +#: The per-backend properties subtree root +#: (``SystemAnalysisSpec.properties_key``). +PROPERTIES_KEY = KeyFamily(type(None), Properties, _slot_name) + + +def _component_key(feat: FeatureUnit, plugin_digest: str | None) -> str: + raw_digest = component_digest(feat) + if plugin_digest is not None: + raw_digest += f"-{plugin_digest}" + return raw_digest + +#: One unit's extraction subtree; the whole subtree moves when the active +#: plugin set changes (``plugins.manifest_digest``). +COMPONENT_KEY = KeyFamily(Properties, ComponentGroup, _component_key) + +def _final_properties_key( + threat_model_digest: str | None, + with_refinement: bool, + extra_context_digest: str | None = None, +) -> str: + # Parameterized exactly like BUG_ANALYSIS_KEY: the component namespace is + # shared across runs, so the entry must be keyed by what distinguishes one + # run's extraction from another's — a single fixed leaf would be + # last-write-wins across divergent runs. + base_key = "final_props" + if with_refinement: + base_key += "|refine" + if threat_model_digest is not None: + base_key += "-tm-" + threat_model_digest + if extra_context_digest is not None: + base_key += "-xc-" + extra_context_digest + return base_key + +#: The property batch as it left the property pipeline (post-inference plugin +#: rewrites applied) plus the tool-contributing plugin ids — the exact +#: derivation inputs of the FORMALIZATION_KEY sibling. Written by the driver; +#: read by offline walkers (``composer.meta.run``) to reconstruct that edge. +FINAL_PROPERTIES_KEY = KeyFamily(ComponentGroup, FinalProperties, _final_properties_key) + + +def _props_and_plugins(props: list[PropertyFormulation], plugins: list[str] | None = None) -> str: + if not plugins: + return _props_digest(props) + return _props_digest(props) + "-" + string_hash("|".join(plugins)) + +#: One unit's formalization result, keyed by the exact property batch. The +#: child is the backend's result type — pass ``formalizer.formalized_type``. +FORMALIZATION_KEY = PolyKeyFamily(ComponentGroup, _props_and_plugins) + +#: The verification artifacts a batch's plugin tools registered, cached under +#: the formalization child so cache replays (where the tools never run) still +#: carry them into the report. Parent is the backend's result-typed namespace, +#: which no static declaration can name — hence ``Any``. +PLUGIN_ARTIFACTS_KEY = CacheKey[Any, RegisteredArtifacts]("plugin-artifacts") + + +def _plugin_formalization_key(plugin: str): + return f"formalization-plugin-{plugin}" + +PLUGIN_FORMALIZATION_KEY = KeyFamily(ComponentGroup, FormalizationTool, _plugin_formalization_key) + +def _pre_property_key(feat: FeatureUnit, plugin: str) -> str: + return f"{component_digest(feat)}-{string_hash(plugin)}-pre" + +#: A plugin's private pre-inference namespace: sibling of the unit's +#: COMPONENT_KEY subtree, one per (unit, plugin). +PRE_PROPERTY_KEY = KeyFamily(Properties, PrePropertyInference, _pre_property_key) + + +def _post_property_key( + feat: FeatureUnit, plugin: str, curr_props: list[PropertyFormulation] +) -> str: + return f"{component_digest(feat)}-{string_hash(plugin)}-{_props_digest(curr_props)}" + +#: A plugin's post-inference namespace, additionally keyed by the property +#: list entering the hook (each plugin in the post chain sees — and is keyed +#: by — its predecessor's output). +POST_PROPERTY_KEY = KeyFamily(Properties, PostPropertyInference, _post_property_key) diff --git a/composer/pipeline/plugin_api.py b/composer/pipeline/plugin_api.py index 0e4f106b..8d698bf9 100644 --- a/composer/pipeline/plugin_api.py +++ b/composer/pipeline/plugin_api.py @@ -1,23 +1,30 @@ -from typing import AsyncContextManager, Protocol, Callable, Awaitable, Any +from typing import AsyncContextManager, Protocol, Callable, Awaitable, Any, Sequence from dataclasses import dataclass from functools import cached_property from abc import ABC, abstractmethod -from graphcore.graph import TemplateLoader + +from langchain_core.tools import BaseTool + +from graphcore.graph import TemplateLoader, PromptInput from jinja2.loaders import BaseLoader, PackageLoader, ChoiceLoader, PrefixLoader from composer.templates.loader import base_loader, load_jinja_template, _autoescape from composer.spec.system_model import FeatureUnit -from composer.pipeline.ptypes import PipelineRun from composer.spec.context import WorkflowContext, SourceCode from composer.spec.service_host import ServiceHost -from composer.spec.types import PropertyFormulation +from composer.spec.types import PropertyFormulation, VerificationArtifact from composer.spec.prop_inference import AnyPropertyGenerationInput -class PluginContext[C](Protocol): +class PluginRunContext[C](Protocol): + """The run's services, without the task runner: what a tool-binding hook gets. + Tool hooks bind tools into the backend's authoring graph — they don't run + top-level tasks, so the runner capability lives only on :class:`PluginContext`, + the task-hook context.""" + @property def ctx(self) -> WorkflowContext[C]: ... - + @property def env(self) -> ServiceHost: ... @@ -26,6 +33,7 @@ def env(self) -> ServiceHost: def source(self) -> SourceCode: ... +class PluginContext[C](PluginRunContext[C], Protocol): async def runner[T]( self, label: str, @@ -33,12 +41,37 @@ async def runner[T]( ) -> T: ... + +class ArtifactRegistrar(Protocol): + """How a plugin's contributed tool reports a verification artifact it + produced (a Lean proof, an auxiliary certificate, …). Registration is + in-memory and synchronous — the driver persists registered artifacts via + the run's artifact store and records them in the report, attributed to the + registering plugin. Call it from tool bodies as they produce results; the + driver collects per formalization batch.""" + + def register(self, artifact: VerificationArtifact) -> None: + ... + + +class PluginToolContext[C](PluginRunContext[C], Protocol): + """What a tool-binding hook receives: the run's services plus the artifact + registrar for the batch being formalized. Still no task runner — tool hooks + bind tools, they don't run top-level tasks.""" + + @property + def artifacts(self) -> ArtifactRegistrar: + ... + class PrePropertyInference: pass class PostPropertyInference: pass +class FormalizationTool: + pass + import jinja2 from jinja2 import Environment, ChoiceLoader, PrefixLoader @@ -68,8 +101,43 @@ def join_path(self, template, parent): return prefix + template return template +@dataclass +class ProvidedTools: + tools: Sequence[BaseTool] + system_prompt_injection: PromptInput + +def plugin_jinja_loader( + loader: BaseLoader | None | type +) -> TemplateLoader: + if loader is None: + return load_jinja_template + + if isinstance(loader, type): + try: + loader = PackageLoader(loader.__module__) + except ValueError: + return load_jinja_template + + new_jinja_loader = ChoiceLoader([ + loader, + _NonStrippingPrefixLoader({ + "autoprover": base_loader + }) + ]) + compilation_env = _PluginEnvironment(loader=new_jinja_loader, autoescape=_autoescape) + def _load_jinja_template(template_name: str, **kwargs: Any) -> str: + """Load and render a Jinja template from the script directory""" + template = compilation_env.get_template(template_name) + return template.render(**kwargs) + return _load_jinja_template + + class PipelinePlugin[U: FeatureUnit](ABC): - """A pipeline plugin, parameterized by the unit type its hooks accept.""" + """A pipeline plugin, parameterized by the unit type its hooks accept. Tool + contribution is not declared here: a plugin opts in by deriving from a + tool-provider subclass instead of this base directly — a backend's own + (``composer.spec.source.plugin.CertoraProverTools``, + ``composer.foundry.plugin.FoundryTools``) or :class:`AnyBackendTools` below.""" NAME: str @@ -79,27 +147,11 @@ def plugin_loader(self) -> BaseLoader | None: return PackageLoader(t) except ValueError: return None - + @cached_property def load_jinja_template(self) -> TemplateLoader: loader = self.plugin_loader() - if loader is None: - return load_jinja_template - - - new_jinja_loader = ChoiceLoader([ - loader, - _NonStrippingPrefixLoader({ - "autoprover": base_loader - }) - ]) - compilation_env = _PluginEnvironment(loader=new_jinja_loader, autoescape=_autoescape) - def _load_jinja_template(template_name: str, **kwargs: Any) -> str: - """Load and render a Jinja template from the script directory""" - template = compilation_env.get_template(template_name) - return template.render(**kwargs) - return _load_jinja_template - + return plugin_jinja_loader(loader) async def property_inference_input_hook( self, @@ -117,6 +169,41 @@ async def post_process_property_inference( return props +# --------------------------------------------------------------------------- +# Tool contribution — deriving a tool-provider subclass IS the declaration +# --------------------------------------------------------------------------- +# +# A plugin contributes formalization tools by deriving from a tool-provider +# subclass of PipelinePlugin: each backend defines its own in its own package +# (the prover's ``composer.spec.source.plugin.CertoraProverTools``, foundry's +# ``composer.foundry.plugin.FoundryTools``) and hands it to the driver as a +# ``ToolExtension`` (see ``composer.pipeline.core``) — this module stays +# backend-agnostic. The driver dispatches — and perturbs the formalization +# cache key for — exactly the plugins deriving the running backend's provider +# class, so declaration and implementation cannot drift, and a plugin that +# derives none (property-inference-only plugins) leaves every formalization +# key plugin-free. Each hook returns one tool bundle, or None when it has +# nothing for this particular unit/batch (None keeps the plugin in the cache +# key: whether it yields is only knowable at dispatch time). + + +class AnyBackendTools[U: FeatureUnit](PipelinePlugin[U]): + """The one backend-agnostic tool provider: the same tools for every backend, + present and future — so no staged backend state, since none is common across + backends. Always projected by the driver's binder, so it perturbs every + backend's formalization keys; prefer a backend's own provider class when the + tools are backend-specific. Composes with those: on a backend whose provider + class the plugin also derives, both hooks fire and both bundles are injected.""" + + @abstractmethod + async def backend_tools( + self, + comp: U, + prop: Sequence[PropertyFormulation], + tool_context: PluginToolContext[FormalizationTool], + ) -> ProvidedTools | None: + ... + # --------------------------------------------------------------------------- # Scope — which runs a plugin applies to # --------------------------------------------------------------------------- diff --git a/composer/pipeline/plugins.py b/composer/pipeline/plugins.py index f66bca9f..a262c533 100644 --- a/composer/pipeline/plugins.py +++ b/composer/pipeline/plugins.py @@ -13,21 +13,40 @@ ) from composer.spec.service_host import ServiceHost from composer.spec.system_model import FeatureUnit -from composer.spec.util import string_hash +from composer.spec.util import combine_digests, string_hash from .ptypes import PipelineRun from .plugin_api import ( - AnyEcosystem, ForEcosystem, PipelinePluginLoader, PipelinePlugin, PluginContext, PluginScope, + AnyEcosystem, ArtifactRegistrar, ForEcosystem, PipelinePluginLoader, PipelinePlugin, + PluginContext, PluginRunContext, PluginScope, PluginToolContext, ) class _RunnerFun(Protocol): async def __call__[T](self, label: str, job: Callable[[], Awaitable[T]]) -> T: ... + +def _with_plugin_loader(env: ServiceHost, plugin: PipelinePlugin[Any]) -> ServiceHost: + """The run's services with the plugin's own template loader swapped in, so the + plugin's prompts resolve against its package (falling back to the autoprover + namespace — see ``plugin_api.plugin_jinja_loader``).""" + return replace(env, models=replace(env.models, _loader=plugin.load_jinja_template)) + + @dataclass -class PluginRunner[C]: +class _PluginServices[C]: + """The runner-less :class:`PluginRunContext` implementation.""" ctx: WorkflowContext[C] env: ServiceHost source: SourceCode + +@dataclass +class _PluginToolServices[C](_PluginServices[C]): + """The :class:`PluginToolContext` implementation — what tool-binding hooks + receive: run services plus the batch's artifact registrar.""" + artifacts: ArtifactRegistrar + +@dataclass +class PluginRunner[C](_PluginServices[C]): runner: _RunnerFun class DisplayStrings(tuple[str, str]): @@ -50,7 +69,7 @@ def __new__( class PluginPhaseRunner[P: enum.Enum, U: FeatureUnit]: plugin: PipelinePlugin[U] _run: PipelineRun[P, Any] - _phase: tuple[P, str] + _phase: P _sub_phase: DisplayStrings plugin_id: str @@ -59,21 +78,18 @@ def bind[C]( uid: str, ctxt: WorkflowContext[C] ) -> PluginContext[C]: - new_loader = self.plugin.load_jinja_template - env = self._run.env - env = replace(env, models=replace(env.models, _loader=new_loader)) async def run[T]( label: str, job: Callable[[], Awaitable[T]] ) -> T: label = f"(Plugin {self._sub_phase.display_str}) {self.plugin.NAME}: {label}" return await self._run.runner( - TaskInfo(f"{self._phase[0].name}-{self._sub_phase.id_str}-{self.plugin_id}-{uid}", label, self._phase[0]), + TaskInfo(f"{self._phase.name}-{self._sub_phase.id_str}-{self.plugin_id}-{uid}", label, self._phase), job, ) return PluginRunner( ctxt, - env, + _with_plugin_loader(self._run.env, self.plugin), self._run.source, run ) @@ -121,9 +137,7 @@ def applicable_plugin_manifest[U: FeatureUnit](unit_type: type[U]) -> list[str]: def manifest_digest(manifest: list[str]) -> str | None: """Digest of a sorted plugin manifest as suffixed onto per-component cache keys; ``None`` when no plugins are active.""" - if not manifest: - return None - return string_hash("|".join(manifest)) + return combine_digests(manifest) @dataclass @@ -141,16 +155,34 @@ def plugin_digest(self) -> None | str: def plugin_manifest(self) -> list[str]: return sorted(self._plugins.keys()) + def sorted_plugins(self) -> Iterable[tuple[str, PipelinePlugin[U]]]: + """Deterministic ``(plugin id, plugin)`` iteration — the order tool + contributions are bound in (injection order is prompt content).""" + return sorted(self._plugins.items(), key=lambda r: r[0]) + + def tool_context[C]( + self, + plugin: PipelinePlugin[U], + ctxt: WorkflowContext[C], + artifacts: ArtifactRegistrar, + ) -> PluginToolContext[C]: + """The context a tool-binding hook receives: the run's services under the + plugin's template loader, the batch's artifact registrar, and deliberately + NO task runner — tool hooks bind tools, they don't run top-level tasks.""" + return _PluginToolServices( + ctxt, _with_plugin_loader(self._run.env, plugin), self._run.source, artifacts + ) + def bind_phase( - self, phase: P, label: str + self, phase: P ) -> "PluginPhaseManager[P, U]": return PluginPhaseManager( - self._plugins, self._run, (phase, label) + self._plugins, self._run, phase ) @dataclass class PluginPhaseManager[P: enum.Enum, U: FeatureUnit](PluginManager[P, U]): - _phase: tuple[P, str] + _phase: P def runners(self, *, sub_phase_id: str, sub_phase_label: str, sorted_run: bool = False) -> Iterable[PluginPhaseRunner[P, U]]: to_iter : Iterable[tuple[str, PipelinePlugin[U]]] = sorted( diff --git a/composer/pipeline/ptypes.py b/composer/pipeline/ptypes.py index fbf1a18a..413a48c9 100644 --- a/composer/pipeline/ptypes.py +++ b/composer/pipeline/ptypes.py @@ -15,7 +15,9 @@ from composer.spec.system_model import ( FeatureUnit, ) -from composer.spec.types import PropertyFormulation, FormalResult +from composer.spec.types import ( + Curtailed, PropertyFormulation, FormalResult, VerificationArtifact, +) from composer.spec.source.report.collect import ReportableResult @@ -28,12 +30,19 @@ class GaveUp(BaseModel): spec.source.author and foundry.author).""" reason: str +#: Default width of the CPU budget (``--max-cpu-tasks``). Each such task is a toolchain +#: invocation that already parallelizes across cores, so a second concurrent one is about all a +#: developer machine absorbs before the two only contend with each other. +DEFAULT_MAX_CPU_TASKS = 2 + + @dataclass class TaskRunnerHost[P: enum.Enum, H, S: SourceFields, C]: ctx: WorkflowContext[C] source: S _handler_factory: HandlerFactory[P, H] - _semaphore: asyncio.Semaphore + _agent_semaphore: asyncio.Semaphore + _cpu_semaphore: asyncio.Semaphore async def runner[T]( self, @@ -44,7 +53,28 @@ async def runner[T]( factory=self._handler_factory, fn=job, info=task_info, - semaphore=self._semaphore + semaphore=self._agent_semaphore + ) + + async def cpu_runner[T]( + self, + task_info: TaskInfo[P], + job: Callable[[], Awaitable[T]] | Callable[[ConversationContextProvider], Awaitable[T]], + ) -> T: + """:meth:`runner` for a task that is not an agent — a toolchain build, say. + + A run has two budgets. The agent semaphore (``--max-concurrent``, default 4) bounds + concurrent *model* work; the CPU semaphore (``--max-cpu-tasks``) bounds work that spends + cores and wall-clock instead. Charging a build to the agent budget would quietly take away + concurrency the user asked for — a ten-minute cargo build would hold one of the four slots + for the whole phase it overlaps — but leaving it unbounded is no better once there is more + than one such task, since two toolchains on the same machine just contend. The task is + otherwise identical: same handler, phase, and lifecycle events, a different budget.""" + return await run_task( + factory=self._handler_factory, + fn=job, + info=task_info, + semaphore=self._cpu_semaphore ) # ---- run-scoped shared infra, handed to every hook --------------------------- @@ -80,10 +110,47 @@ class BackendJob[U: FeatureUnit]: feat: U props: list[PropertyFormulation] + +class FinalProperties(BaseModel): + """A component's property batch as it left the property pipeline — after + every post-inference plugin rewrite — plus the tool-contributing plugin + ids the driver gated in. Together these are exactly the inputs + ``FORMALIZATION_KEY`` is derived from, so an offline walker + (``composer.meta.run``) can reconstruct the formalization edge without + sniffing the store. Written by the driver at formalization time; the + pre-rewrite batch remains ``_BugAnalysisCache``.""" + items: list[PropertyFormulation] + tool_plugins: list[str] + +class PluginArtifact(BaseModel): + """One registered verification artifact with its contributing plugin's id — + the driver stamps the attribution, so plugins never handle their own id.""" + plugin: str + artifact: VerificationArtifact + + +class RegisteredArtifacts(BaseModel): + """The artifacts a batch's plugin tools registered during formalization, + cached under the formalization namespace so cache replays (where the tools + never run) still carry them into the report.""" + items: list[PluginArtifact] + + +@dataclass(frozen=True) +class PersistedPluginArtifact: + """A registered artifact after the store wrote it: the registration plus + the project-relative path the report records.""" + plugin: str + artifact: VerificationArtifact + path: Path + + @dataclass(frozen=True) class Delivered[FormT: BackendResult]: - """A successful formalization and the project-relative path it was persisted to. The path exists - only because the result does, so the two travel together rather than as independent fields.""" + """A formalization result and the project-relative path it was persisted to. The path exists + only because the result does, so the two travel together rather than as independent fields. + On its own this means a *successful* formalization; wrapped in a `Curtailed` it is the + quarantined partial a budget-cut author managed to publish.""" result: FormT deliverable: Path @@ -100,7 +167,11 @@ def run_link(self) -> str | None: @dataclass class ComponentOutcome[FormT: BackendResult, U: FeatureUnit](BackendJob[U]): - result: Delivered[FormT] | GaveUp | BaseException + result: Delivered[FormT] | GaveUp | BaseException | Curtailed[Delivered[FormT]] + #: Verification artifacts the batch's plugin tools registered, already + #: persisted by the artifact store. Independent of ``result``: a component + #: that gave up may still have produced artifacts worth reporting. + artifacts: list[PersistedPluginArtifact] = field(default_factory=list) @dataclass class CorePipelineResult[FormT: BackendResult]: @@ -113,20 +184,42 @@ class CorePipelineResult[FormT: BackendResult]: @property def n_delivered(self) -> int: - """Components that produced a deliverable — a successful formalization. Everything - else (a ``GaveUp`` or a crash) is a component that failed to generate.""" + """Components that produced a deliverable — a successful formalization. Everything else + (a ``GaveUp``, a budget ``Curtailed``, or a crash) is a component with no reliable + result.""" return sum(1 for o in self.outcomes if isinstance(o.result, Delivered)) @property def all_failed(self) -> bool: - """Every attempted component failed to generate or gave up - te run is a total failure. - Guarded on a non-empty outcome set so "all of nothing" is never reported as failure (the - driver raises before returning in the no-outcomes case anyway).""" + """Every attempted component ended without a reliable deliverable (gave up, crashed, or + was budget-curtailed) — the run is a total failure. Guarded on a non-empty outcome set so + "all of nothing" is never reported as failure (the driver raises before returning in the + no-outcomes case anyway).""" return bool(self.outcomes) and self.n_delivered == 0 +class PhaseBudget(TypedDict): + """Per-phase spending *caps* (USD). Ceilings, not allotments: they bound + how much a single phase may hog, and need not sum to the run total — + unspent phase money stays in the run pool for later phases.""" + formalization_preparation: float + system_analysis: float + system_preparation: float + property_extraction: float + formalization: float + + +@dataclass(frozen=True) +class RunBudget: + """The run's token-cost budget: ``total`` is the pool (the real bound on + overall spend); ``caps`` are the per-phase ceilings drawn against it.""" + total: float + caps: PhaseBudget + __all__ = [ + "DEFAULT_MAX_CPU_TASKS", "CorePipelineResult", "ComponentOutcome", + "Curtailed", "Delivered", "BackendJob", "SystemAnalysisSpec", diff --git a/composer/pipeline/run_tags.py b/composer/pipeline/run_tags.py index 9eb10106..eec4381b 100644 --- a/composer/pipeline/run_tags.py +++ b/composer/pipeline/run_tags.py @@ -35,6 +35,11 @@ class AutoProveCacheTags(BaseModel): """``Document.to_digest()`` of the threat model, which parameterizes ``bug_analysis_key``; ``None`` for runs without one.""" + extra_context_digests: list[str] = Field(default_factory=list) + """``Document.to_digest()`` of each ``--extra-context`` document, in order. Folded + through ``combine_digests`` to parameterize ``bug_analysis_key``; kept unfolded here + so the individual documents stay identifiable.""" + interactive: bool | None = None """Whether the run used interactive refinement (selects the ``|refine`` bug-analysis key variant). ``None`` on records written before this diff --git a/composer/prover/agentic_analyzer.py b/composer/prover/agentic_analyzer.py index 39531a2e..70690316 100644 --- a/composer/prover/agentic_analyzer.py +++ b/composer/prover/agentic_analyzer.py @@ -29,6 +29,7 @@ import asyncio from dataclasses import dataclass +import logging from pathlib import Path from typing import Annotated, Literal, NotRequired, Union, override import uuid @@ -57,6 +58,8 @@ from composer.prover.cex_task_ids import cex_rule_task_id, cex_aggregator_task_id from composer.tools.thinking import RoughDraftState, get_rough_draft_tools +_logger = logging.getLogger(__name__) + # --------------------------------------------------------------------------- # Per-CEX commit shape (discriminated union) @@ -490,12 +493,28 @@ async def _process_rule_addressed( with set_current_task_id(cex_rule_task_id(tool_call_id, rule_group.rule_name)): return await _process_rule(rule_group) + # A rule whose analysis raises forfeits its own root causes only; the + # remaining rules still report, and the caller still gets a rendered + # report rather than losing the prover run. per_rule = await asyncio.gather( - *(_process_rule_addressed(rg) for rg in failing_rules) + *(_process_rule_addressed(rg) for rg in failing_rules), + return_exceptions=True, ) + for rule_group, outcome in zip(failing_rules, per_rule): + if isinstance(outcome, asyncio.CancelledError): + raise outcome + if isinstance(outcome, BaseException): + _logger.warning( + "CEX analysis failed for rule %s, continuing without its root causes: %r", + rule_group.rule_name, + outcome, + ) # Flatten preserving rule order (gather preserves input order). all_causes: list[_PerRuleRootCause] = [ - c for rule_results in per_rule for c in rule_results + c + for rule_results in per_rule + if not isinstance(rule_results, BaseException) + for c in rule_results ] if not all_causes: diff --git a/composer/prover/core.py b/composer/prover/core.py index cf1a492b..76b1876d 100644 --- a/composer/prover/core.py +++ b/composer/prover/core.py @@ -32,6 +32,7 @@ import json import logging import os +import uuid from langchain_core.messages import AnyMessage, HumanMessage @@ -45,7 +46,7 @@ from composer.prover.analysis import analyze_cex_raw from composer.prover.cloud import CloudJobError, cloud_results -from composer.prover.ptypes import RuleResult +from composer.prover.ptypes import RuleResult, RulePath, StatusCodes from composer.prover.results import read_and_format_run_result from composer.templates.loader import load_jinja_template from composer.prover.prover_protocol import ProverResult @@ -117,10 +118,26 @@ class ProverReport: them through the return value. ``link`` is the prover run's URL (cloud) or local results directory. + + ``certora_run_stdout`` is the captured stdout of the ``certoraRun`` + invocation. It carries diagnostic signal that never reaches the rule + results — e.g. internal function summarization silently failing on + stack-too-deep — so it rides along even on successful runs. """ - rule_status: dict[str, bool] + raw_rule_status: dict[RulePath, StatusCodes] + result_str: str link: str + certora_run_stdout: str + + @property + def rule_status(self) -> dict[str, bool]: + to_ret = {} + for (k, v) in self.raw_rule_status.items(): + if k.rule in to_ret and not to_ret[k.rule]: + continue + to_ret[k.rule] = v == "VERIFIED" + return to_ret @property def all_verified(self) -> bool: @@ -299,8 +316,25 @@ async def _one(instance: RuleResult) -> tuple[RuleResult, str | None]: await callbacks.on_analysis_complete(instance, analysis) return (instance, analysis) - jobs = [_one(r) for r in all_results if r.status == "VIOLATED"] - results = await asyncio.gather(*jobs) + violated = [r for r in all_results if r.status == "VIOLATED"] + # One counterexample's analysis failing is not a reason to lose the + # prover run that produced it: the rule keeps its status and only its + # explanation goes missing, so the report renders without it. + settled = await asyncio.gather( + *(_one(r) for r in violated), return_exceptions=True + ) + results: list[tuple[RuleResult, str | None]] = [] + for rule, outcome in zip(violated, settled): + if isinstance(outcome, asyncio.CancelledError): + raise outcome + if isinstance(outcome, BaseException): + _logger.warning( + "CEX analysis failed for rule %s, continuing without its explanation: %r", + rule.name, + outcome, + ) + continue + results.append(outcome) to_cex_explanation = { r.name: stat for (r, stat) in results if stat is not None @@ -321,8 +355,10 @@ async def _one(instance: RuleResult) -> tuple[RuleResult, str | None]: results=results_for_template, ) + # Counted over every violated rule, not just the analyzed ones, so a + # failed analysis cannot move the summarization threshold. failed_count = sum( - 1 for instance, _ in results + 1 for instance in violated if instance.status != "VERIFIED" ) if failed_count > self.summarization_threshold: @@ -413,6 +449,70 @@ async def run_prover_inner( run_result = cast(ProverResult, json.load(output_file)) return run_result, stdout +async def declared_rules_list( + folder: Path, + args: list[str] +) -> list[str]: + """ + This is a temporary hack to work around `certoraRun` not providing a "native" + way to list rules. Instead we use certoraRun to build the project, hijack `msg` to find + the generated build dir, and then manually invoke the typechecker with `-listRules` + ourselves against that build dir. + + Not great, obviously, but lets us work on this AP feature while waiting for support for this to + land upstream in certora-cli and the pip distribution channels. + """ + if any(m == "--msg" for m in args): + raise ValueError("This unholy black magic only works if you don't pass msg") + tc_key = uuid.uuid4().hex + proc = await asyncio.subprocess.create_subprocess_exec( + "certoraRun", *args, "--msg", tc_key, "--compilation_steps_only", + cwd=str(folder), + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE + ) + + rc = await proc.wait() + if rc != 0: + raise ValueError("Type check failed?") + from importlib.resources import files + + tc_jar = files("certora_jars") / "Typechecker.jar" + if not tc_jar.is_file(): + raise ValueError("Typechecker not installed") + d = folder / ".certora_internal" + found : Path | None = None + for p in d.iterdir(): + if not p.is_dir(): + continue + is_build_mirror = p / "run.conf" + if not is_build_mirror.is_file(): + continue + try: + payload = json.loads( + is_build_mirror.read_text() + ) + except json.JSONDecodeError: + continue + if not isinstance(payload, dict) or "msg" not in payload or not isinstance(payload["msg"], str): + continue + if payload["msg"] == tc_key: + found = p + break + if found is None: + raise ValueError("Couldn't find build dir") + with tempfile.NamedTemporaryFile("r") as f: + proc = await asyncio.subprocess.create_subprocess_exec( + "java", "-jar", str(tc_jar), "-buildDirectory", str(found), "-typeCheck", "true", "-listRules", f.name, + cwd=str(folder), + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE + ) + tc_rc = await proc.wait() + if tc_rc != 0: + raise ValueError("Nope, no dice") + all_rules = f.read() + return [s for r in all_rules.split() if (s := r.strip()) and s != "envfreeFuncsStaticCheck"] async def run_prover( folder: Path, @@ -528,8 +628,13 @@ async def run_prover( continue prover_report[rule_name] = i.status == "VERIFIED" + raw_rule_results : dict[RulePath, StatusCodes] = { + k.path: k.status for k in parsed.values() + } + return ProverReport( - rule_status=prover_report, + raw_rule_status=raw_rule_results, result_str=result_str, link=run_result["link"], + certora_run_stdout=stdout, ) diff --git a/composer/prover/ptypes.py b/composer/prover/ptypes.py index 04c58aa0..5869d4fc 100644 --- a/composer/prover/ptypes.py +++ b/composer/prover/ptypes.py @@ -64,11 +64,13 @@ class RuleResult: If status == ERROR, error_msg is non-none """ path: RulePath - cex_dump: Optional[str] + cex_dump: str | None status: StatusCodes error_messages: list[str] = field(default_factory=list) + live_check_info : str | None = field(default=None) + @property def name(self) -> str: return self.path.pprint() diff --git a/composer/prover/results.py b/composer/prover/results.py index c8402ef8..84971718 100644 --- a/composer/prover/results.py +++ b/composer/prover/results.py @@ -22,6 +22,7 @@ class RuleNodeModel(BaseModel): status: Optional[str] = Field(description="The smt status") nodeType: str errors: list[RuleNotificationMessages] + LiveCheckInfo: str | None = Field(default=None) class TreeViewStatus(BaseModel): @@ -96,12 +97,18 @@ def flatten_tree_view(context: Path, r: RuleNodeModel, path: RulePath, parent_ty if stat == "ERROR": messages : set[str] = set() _collect_child_errors(r, messages, lambda sev: sev == "error") - return [RuleResult( - path=effective_path, - cex_dump=None, - status=stat, - error_messages=list(messages) - )] + if all( + "timed-out" in err or ("did not run: BlockingCoroutine was cancelled") in err for err in messages + ): + # time out masquerading as an error... + stat = "TIMEOUT" + else: + return [RuleResult( + path=effective_path, + cex_dump=None, + status=stat, + error_messages=list(messages) + )] elif stat == "SKIPPED": warning_message = [ i.message for i in r.errors if i.severity == "error" or i.severity == "warning" @@ -126,8 +133,8 @@ def flatten_tree_view(context: Path, r: RuleNodeModel, path: RulePath, parent_ty )] if stat == "TIMEOUT": - if len(r.children) == 0: - return [RuleResult(path=effective_path, cex_dump=None,status=stat)] + if all(tc.nodeType == "SANITY" for tc in r.children): + return [RuleResult(path=effective_path, cex_dump=None,status=stat, live_check_info=r.LiveCheckInfo)] assert stat == "TIMEOUT" or stat == "VIOLATED" or stat == "SANITY_FAILED" violated_assert_children = any([ c.nodeType == "VIOLATED_ASSERT" for c in r.children]) if violated_assert_children: diff --git a/composer/rag/db.py b/composer/rag/db.py index 1dc3d4c3..0d1549fd 100644 --- a/composer/rag/db.py +++ b/composer/rag/db.py @@ -50,6 +50,14 @@ SANITY_DEFAULT_CONNECTION: str = f"postgresql://extended_rag_user:rag_password@{_RAG_HOST}:{_RAG_PORT}/rag_db" FOUNDRY_DEFAULT_CONNECTION: str = f"postgresql://foundry_rag_user:rag_password@{_RAG_HOST}:{_RAG_PORT}/rag_db" +# Logical knowledge-base tag -> default DB connection, for corpora ingested by the generic importer +# (`composer.scripts.rag_import`). The tag is the one the manifest carries (== a wheel's +# `rag_db_default`), so the import target and the runtime search tools resolve by one name — +# `composer.tools.rag_env` requires both halves before a tag is usable. Empty until the first such +# corpus lands with the application that declares it; the CVL/Foundry builders use the constants +# above instead. +KNOWLEDGE_BASES: dict[str, str] = {} + type _RagHeader = str | None type _ContentHeaders = tuple[_RagHeader, _RagHeader, _RagHeader, _RagHeader, _RagHeader, _RagHeader] diff --git a/composer/rag/import_format.py b/composer/rag/import_format.py new file mode 100644 index 00000000..f6c3b985 --- /dev/null +++ b/composer/rag/import_format.py @@ -0,0 +1,120 @@ +"""The common JSON manifest format for RAG corpora (see ``docs/rag-import-format.md``). + +A *producer* parses a corpus's native docs and emits one :class:`RagManifest` as JSON; the shared +importer (:mod:`composer.scripts.rag_import`) reads that manifest and owns everything downstream — +chunking, embedding, ``part`` numbering, and the DB ingestion. So these models are the seam +between the two halves, and they are deliberately free of any RAG-stack imports (no +``composer.rag.db``, no spaCy): a producer needs only these classes to emit a corpus. + +A manifest carries **two independent products**, one per retrieval index: + +* :attr:`RagManifest.manual_sections` — whole documents for keyword search + exact + ``get_section``. Never split; a section is returned in full. +* :attr:`RagManifest.embedded_groups` — input for the vector index. The importer sub-splits each + group into length-bounded embedded chunks, guided by the per-block kinds. + +The two indexes store different units (documents vs. length-bounded passages), so a producer lays +out each product explicitly — often as two views of the same source, but nothing derives one from +the other. See the design doc for why the products are separate. +""" + +import enum + +from pydantic import BaseModel, Field + +#: The schema version the importer understands. Bumped only on a breaking change; the importer +#: refuses a manifest whose ``version`` it doesn't recognize rather than mis-ingesting it. +SCHEMA_VERSION = 1 + + +class ManualBlockKind(str, enum.Enum): + """The two content kinds a manual section is built from. + + Manual sections are never split, so the only distinction that matters is whether the body is + prose or code — code must stay out of the searchable text.""" + + #: Prose. Lands in the section's content verbatim. + TEXT = "text" + #: A code sample. Held aside in the section's ``code_refs`` with a ```` + #: placeholder in the content, for retrieval to substitute back. + CODE = "code" + + +class ManualBlock(BaseModel): + """One ordered piece of a manual section.""" + + kind: ManualBlockKind + body: str + + +class EmbeddedBlockKind(str, enum.Enum): + """The content kinds an embedded group is built from. + + These carry the chunking semantics the vector index needs: each kind maps 1:1 onto one way of + driving the importer's ``BlockBuilder``, which decides where a length-bounded chunk may be + cut. Producers state what a block *is*; the importer owns how it chunks.""" + + #: A self-contained prose unit (a paragraph). Chunk cuts prefer its boundaries; an overlong + #: body is split at sentence boundaries rather than mid-sentence. + PARAGRAPH = "paragraph" + #: Structure that must survive intact — tables, lists, anything whose lines are not + #: sentences. Never sentence-split: an overlong body becomes one oversized chunk rather than + #: being cut. + ATOMIC = "atomic" + #: Prose that resumes the stream an earlier block interrupted (e.g. the tail of a sentence + #: around an inline code sample). No boundary preference; may be cut at any sentence end. + CONTINUATION = "continuation" + #: A code sample. Stays atomic and is never embedded as prose: the body is held aside in the + #: chunk's ``code_refs`` and a ```` placeholder takes its place in the chunk text. + CODE = "code" + + +class EmbeddedBlock(BaseModel): + """One ordered piece of an embedded group.""" + + kind: EmbeddedBlockKind + body: str + + +class _HeaderPath(BaseModel): + """Shared shape of both products: a header path labelling the content. + + ``headers`` is the ``h1..h6`` path: entry *i* lands in column ``h(i+1)``, and a falsy or + absent level stays ``NULL`` in its own column (the DB's ``_normalize_head`` packs nothing). + At most 6 — a deeper path raises there rather than losing its deepest level. Both retrieval + indexes are keyed off this path.""" + + headers: list[str] + + +class ManualSection(_HeaderPath): + """A whole document for the manual index: keyword search hits it, ``get_section`` returns it + in full under its header path. Any size — the importer never splits it.""" + + blocks: list[ManualBlock] = Field(default_factory=list) + + +class EmbeddedGroup(_HeaderPath): + """A run of blocks the importer chunks together for the vector index. Every resulting + length-bounded chunk is labelled with the group's header path.""" + + blocks: list[EmbeddedBlock] = Field(default_factory=list) + + +class RagManifest(BaseModel): + """A whole corpus: metadata + the two retrieval products, serialized as one JSON document.""" + + #: Schema version; must match :data:`SCHEMA_VERSION`. Defaults so a hand-written manifest can + #: omit it, but the importer still validates any value present. + version: int = SCHEMA_VERSION + #: Logical corpus tag — the *same* string a wheel declares as ``rag_db_default`` and that + #: ``rag_env.py`` resolves to search tools. The importer resolves it to a DB connection via + #: ``composer.rag.db.KNOWLEDGE_BASES`` (overridable by ``--output``). + knowledge_base: str + #: Free-text provenance (source repo/commit/glob). For logs only — not persisted per row + #: (the DB schema is header-only). + source: str | None = None + #: Documents for keyword search + ``get_section``. + manual_sections: list[ManualSection] = Field(default_factory=list) + #: Input for the vector index, chunked by the importer. + embedded_groups: list[EmbeddedGroup] = Field(default_factory=list) diff --git a/composer/rustapp/__init__.py b/composer/rustapp/__init__.py new file mode 100644 index 00000000..d97bb531 --- /dev/null +++ b/composer/rustapp/__init__.py @@ -0,0 +1,211 @@ +"""Host for AutoProver applications whose backend is *implemented* in Rust (PyO3). + +"Rust" here is the **backend implementation** language — the wheel is compiled Rust — and it is +orthogonal to the ecosystem, i.e. the language of the *code being analyzed*: a Rust-implemented +backend may analyze Solidity (``echoprover`` selects ecosystem ``evm``) or Rust (Crucible selects +``solana``). The analyzed-source language rides on the ecosystem (see +``composer.pipeline.ecosystem.Language``); this host never assumes it from the fact that the +backend is a Rust wheel — it reads it from ``app.ecosystem.language``. + +This package is the Python side of the seam described in +``docs/rust-applications.md``. A Rust application is a wheel built with +``autoprover-sdk`` (see ``rust/``) exposing a small, synchronous, JSON FFI surface +— a **passive service** the pipeline drives: + + descriptor() -> str # the AppDescriptor (declarative spine) + validate_preconditions(args_json) -> str|None + target_for(input_json, check) -> str|None # which invocation a declared check runs under + author_prompt(input_json, failure_json|None) -> str + judge(input_json) -> str|None # None ⇒ no judge for this input + judge_instruction(input_json, spec) -> str # one review round's instruction + compile(input_json, spec|None, workdir, sandbox_json) -> str # BLOCKING (run-confined) + validate(input_json, spec, target, workdir, sandbox_json) -> str # BLOCKING (run-confined) + workspace_prep(input_json) -> str # the declarative prep plan the host executes + sandbox_grants(args_json) -> str # extra grants for the host-authored policy + finalize(outcomes_json) -> str|None + +Every one of those strings is a model in :mod:`composer.rustapp.wire` (the runtime ABI) or +:mod:`composer.rustapp.descriptor` (the declarative one) — that is where the JSON shape lives, and +:class:`composer.rustapp.wire.RustAppModule` is the surface above as a type. + +The host loads that module, synthesizes the pipeline's phase enum from the +descriptor, and wraps the module in a :class:`PipelineBackend` whose ``formalize`` +runs the author→compile→judge→validate loop (:mod:`composer.rustapp.adapter`) — +Python owns the loop and every LLM turn; the two blocking callouts run the toolchain +via ``run-confined``. No IoC ``resume`` protocol and no ``pyo3-async`` bridge. + +Entry points: + +* :func:`composer.rustapp.cli.tui_main` / ``console_main`` — a complete runnable + application from a module name (the descriptor drives argparse, the entry point, + the frontend, and ``main()``). This is the whole vertical. +* :func:`composer.rustapp.host.build_application` — synthesize the phase enum, + labels, section order and backend factory for a frontend / ``main()``. +* :func:`composer.rustapp.entry.rust_entry_point` — the async entry point context + manager (services + ``WorkflowContext``), yielding the Executor. +* :func:`composer.rustapp.host.run_rust_pipeline` — headless: build the backend + from a module name and run the shared driver directly. +""" + +from composer.rustapp.descriptor import ( + AppDescriptor, + ArgDefault, + ArgSpec, + ArtifactLayout, + Callout, + DeliverableMode, + EventKind, + PerComponent, + PhaseRole, + PhaseSpec, +) +from composer.rustapp.result import RustArtifact, RustFormalResult +from composer.rustapp.wire import ( + AuthorInput, + CalloutError, + CalloutFailed, + CompileFailed, + CompileOk, + CompileResult, + FinalizeComponent, + FinalizeInput, + Judge, + Prompt, + Property, + RustAppModule, + SandboxGrants, + Check, + SkippedProperty, + ValidateBuildFailed, + ValidateCoverageError, + ValidateOutcome, + ValidateVerdicts, + Verdict, + WorkspacePrep, +) +from composer.rustapp.toolchain import ( + PROJECT_TOOLCHAINS, + ProjectToolchain, + project_toolchain, + source_unit, +) +from composer.rustapp.adapter import ( + PreflightFailed, + ProjectFacts, + RustBackend, + RustFormalizer, + RustPreparedSystem, + RustStagedFormalizer, + UnitProperty, + confined_target, + emit_event, + source_unit_of, + unique_slugs, +) +from composer.rustapp.session import SessionResult, run_session +from composer.rustapp.store import RustArtifactStore +from composer.rustapp.host import ( + BackendOptions, + PhaseModel, + RustApplication, + StoreFactory, + build_application, + build_backend, + build_phase_model, + load_descriptor, + load_module, + resolve_ecosystem, + run_application, + run_rust_pipeline, +) +from composer.rustapp.entry import ( + EnvBuilder, + RustRunner, + build_arg_parser, + build_default_env, + rust_entry_point, +) +from composer.rustapp.frontend import ( + GenericRustApp, + GenericRustConsoleHandler, + GenericRustTaskHandler, +) + +# NOTE: composer.rustapp.cli is intentionally NOT imported here — it runs +# `import composer.bind` (import-time DI / test-tape bootstrap), which the +# built-in apps only trigger from their `main` modules. Import it explicitly: +# from composer.rustapp.cli import tui_main, console_main + +__all__ = [ + "AppDescriptor", + "ArgDefault", + "ArgSpec", + "ArtifactLayout", + "Callout", + "DeliverableMode", + "EventKind", + "PerComponent", + "PhaseRole", + "PhaseSpec", + "RustArtifact", + "RustFormalResult", + "AuthorInput", + "CalloutError", + "CalloutFailed", + "CompileFailed", + "CompileOk", + "CompileResult", + "FinalizeComponent", + "FinalizeInput", + "Judge", + "Prompt", + "Property", + "RustAppModule", + "SandboxGrants", + "Check", + "SkippedProperty", + "ValidateBuildFailed", + "ValidateCoverageError", + "ValidateOutcome", + "ValidateVerdicts", + "Verdict", + "WorkspacePrep", + "PROJECT_TOOLCHAINS", + "ProjectToolchain", + "project_toolchain", + "source_unit", + "PreflightFailed", + "ProjectFacts", + "RustBackend", + "RustFormalizer", + "RustPreparedSystem", + "RustStagedFormalizer", + "UnitProperty", + "SessionResult", + "run_session", + "confined_target", + "emit_event", + "source_unit_of", + "unique_slugs", + "RustArtifactStore", + "RustApplication", + "BackendOptions", + "PhaseModel", + "StoreFactory", + "build_application", + "build_backend", + "build_phase_model", + "load_descriptor", + "load_module", + "resolve_ecosystem", + "run_application", + "run_rust_pipeline", + "EnvBuilder", + "RustRunner", + "build_arg_parser", + "build_default_env", + "rust_entry_point", + "GenericRustApp", + "GenericRustConsoleHandler", + "GenericRustTaskHandler", +] diff --git a/composer/rustapp/adapter.py b/composer/rustapp/adapter.py new file mode 100644 index 00000000..87436f98 --- /dev/null +++ b/composer/rustapp/adapter.py @@ -0,0 +1,748 @@ +"""Adapter: wrap a Rust wheel (a :class:`~autoprover_sdk.Backend`) as a +:class:`~composer.pipeline.core.PipelineBackend`. + +The Rust wheel is a **passive service** (``docs/rust-applications.md``): Python owns every LLM turn +and calls the wheel's pure callouts (``descriptor`` / ``target_for`` / ``author_prompt`` / +``check_syntax`` / ``judge`` / ``finalize``) plus the two blocking ones (``compile`` / +``validate``) that run the toolchain via ``run-confined``. There is no IoC ``resume`` loop and no +``Effects`` protocol. + +Authoring itself is the shared session of :mod:`composer.authoring`, assembled for a wheel in +:mod:`composer.rustapp.session`: the agent owns a spec buffer, the wheel's ``validate`` is a tool it +calls, and publishing is gated on stamps over the buffer as it stands. This module is what connects +that session to the pipeline — phases, preflight, the setup spec's cache, and the report. + +Three phase objects mirror the CVL / foundry backends: + +* :class:`RustBackend` — ``PipelineBackend`` (guidance, phases, store, ``preflight`` / + ``prepare_system``). +* :class:`RustPreparedSystem` — builds the formalizer (thin; no app-specific setup). +* :class:`RustFormalizer` — ``formalize`` runs the loop; ``fetch_verdicts`` reads the verdicts + ``validate`` baked into the result. + +App-specific orchestration (a shared setup spec, workspace prep + its gate, crate assembly) is +descriptor-driven here — no per-application Python package (``docs/rust-applications.md``): the wheel +declares ``preflight`` / ``setup`` / ``workspace_prep`` / ``deliverable_mode=callout`` / ``finalize`` +and the generic host runs them. + +The two build-shaped steps are overlapped with the LLM steps that don't need them: +:meth:`RustBackend.preflight` (prepare the workspace, then *gate* it — a wheel-authored skeleton +built by the real toolchain) runs alongside system analysis, and the shared setup spec is +authored after extraction, when the properties it must make checkable finally exist. +""" + +import asyncio +import enum +import json +import logging +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any, Awaitable, Callable, override, Sequence + + +from langgraph.config import get_stream_writer + +from composer.io.multi_job import TaskInfo +from composer.pipeline.core import ( + BackendJob, + ComponentOutcome, + CorePhases, + Delivered, + Formalizer, + GaveUp, + PipelineRun, + PreparedSystem, + StagedFormalizer, + SystemAnalysisSpec, + ToolBinder +) +from composer.pipeline.ecosystem import ChainTag, Ecosystem +from composer.sandbox.command import DEFAULT_TIMEOUT_S +from composer.sandbox.config import BackendSpec, SandboxConfig +from composer.rustapp.descriptor import AppDescriptor, PhaseRole, PhaseSpec +from composer.rustapp.phases import PhaseModel +from composer.rustapp.result import RustArtifact, RustFormalResult, RustSetupSpec +from composer.rustapp.toolchain import project_toolchain, source_unit +from composer.rustapp.session import ( + run_session, +) +from composer.rustapp.wire import ( + AuthorInput, + CompileOk, + ComponentGaveUp, + ComponentInput, + FinalizeComponent, + FinalizeInput, + PreflightInput, + Property, + RustAppModule, + SetupInput, + parse_compile, + parse_files, + parse_workspace_prep, +) +# The wheel's per-check verdict and the *report's* per-check verdict are different types with the same +# name (``fetch_verdicts`` maps one to the other), so the wire one is aliased here. Likewise +# ``Delivered``: the pipeline's component outcome and the wire's payload for one. +from composer.rustapp.wire import Delivered as WireDelivered +from composer.rustapp.wire import SkippedProperty as WireSkipped +from composer.spec.artifacts import ArtifactStore +from composer.spec.context import SourceFields, WorkflowContext +from composer.spec.key_family import KeyFamily +from composer.spec.source.report.collect import Formalized, ReportComponentInput, Verdict +from composer.spec.source.report.schema import RuleName +from composer.spec.system_model import BaseApplication, FeatureUnit +from composer.spec.types import ComponentName, PropertyFormulation +from composer.spec.util import slugify_filename, string_hash + +_log = logging.getLogger(__name__) + + +# --------------------------------------------------------------------------- +# Shared loop helpers (used by RustFormalizer.formalize and app setup specs). +# --------------------------------------------------------------------------- + +def emit_event(kind: str, payload: dict) -> None: + """Stream a domain event to the current task's panel, as the wheel's ``{"type": kind, …}`` + custom-stream payload. + + Callable only from inside a graph run — every emission site is a gate tool's body, where + LangGraph's stream writer routes the event with the run's own thread and checkpoint.""" + get_stream_writer()({"type": kind, **payload}) + + +@dataclass(frozen=True) +class UnitProperty: + """A property and the unit whose analysis produced it — the two halves of what identifies a + property across a run (the report's ``PropertyKey``). + + Titles are unique only within a unit, so wherever properties from more than one unit meet — the + shared setup spec's input — they travel paired with their unit rather than as bare + formulations.""" + + component: ComponentName + prop: PropertyFormulation + + +def unique_slugs(props: list[PropertyFormulation]) -> list[str]: + """One unique kebab slug per property — what a wheel names that property's check after. A + collision gets a numeric suffix, whether it comes from punctuation/casing or from two units + genuinely sharing a title (the setup spec's input spans units, where that is allowed). + + Not the artifact's name: a deliverable file is named from the *component's* slug + (:meth:`RustBackend.to_artifact_id`), which is a different thing entirely.""" + slugs: list[str] = [] + seen: dict[str, int] = {} + for p in props: + base = slugify_filename(p.title) or "inv" + n = seen.get(base, 0) + seen[base] = n + 1 + slugs.append(base if n == 0 else f"{base}_{n}") + return slugs + + +def _properties(owned: Sequence[UnitProperty]) -> list[Property]: + """The wire form of the properties one spec must make checkable — each naming the unit it was + inferred for, and the host-assigned slug a wheel names its check after (see + :func:`unique_slugs`).""" + return [ + Property( + component=o.component, title=o.prop.title, sort=o.prop.sort, + description=o.prop.description, slug=slug, + ) + for o, slug in zip(owned, unique_slugs([o.prop for o in owned])) + ] + + +def confined_target(root: Path, rel: str) -> Path: + """Join a wheel-supplied relative path under ``root``, rejecting absolute paths / ``..`` + traversal — mirrors the Rust ``confined_join`` so host-written deliverable/prep files stay + inside the project (the wheel is trusted, but defense-in-depth is cheap). + + Public because the toolchain half of a workspace prep lives outside this module + (:class:`~composer.rustapp.toolchain.ProjectToolchain`) and whatever it writes must be confined + exactly as the host's own writes are.""" + p = Path(rel) + if p.is_absolute() or ".." in p.parts: + raise ValueError(f"unsafe file path {rel!r}: absolute or traverses outside the workdir") + return root / p + + +def source_unit_of( + ecosystem: Ecosystem[Any, Any, Any], source: SourceFields +) -> dict[str, Any]: + """The ``AuthorInput.source_unit`` field — where the code under analysis lives as a unit of its + own build system — from the chain's registered toolchain + (:func:`composer.rustapp.toolchain.source_unit`). + + A wheel that must *depend on* the analyzed code (Crucible's harness path-depends on the program + under test) reads this instead of deriving a directory or package name from + ``source.contract_name``, which is only the analysis identifier. Empty when the chain has no + toolchain, when the language has no such unit (Solidity), or when the layout couldn't be read — + all three mean the same thing to the wheel, which then applies its own convention. + """ + return source_unit(ecosystem.name, source) + + +def _setup_identity(input: SetupInput) -> str: + """A cache key for the shared setup spec: a hash of what it is authored *from*. + + Exactly the inputs the wheel renders the artifact from — the program, the project facts (where its + code lives, and what the prep established, which is what decides where its types come from), the + analyzed model, and the properties it has to make checkable. Deliberately NOT the whole input: + ``args`` also carries run knobs (a fuzz budget) that don't change what gets authored, and keying + on those would throw the artifact away for no reason. + """ + material = { + "program": input.program, + "source_unit": input.source_unit, + "prep_facts": input.prep_facts, + "model": input.model, + "props": [p.model_dump() for p in input.props], + } + return string_hash(json.dumps(material, sort_keys=True, default=str)) + + +def _setup_key(descriptor_name: str, input: SetupInput) -> str: + return f"{descriptor_name}-setup-{_setup_identity(input)}" + + +#: The shared setup spec's slot under the run root, keyed per wheel and per +#: :func:`_setup_identity` of what the artifact is authored from. +RUST_SETUP_KEY = KeyFamily(type(None), RustSetupSpec, _setup_key) + + +async def run_workspace_prep( + module: RustAppModule, + input: AuthorInput, + *, + chain: ChainTag, + source: SourceFields, + sandbox: SandboxConfig | None, + command_timeout_s: int, +) -> dict[str, Any]: + """Execute the wheel's pure ``workspace_prep`` plan (``docs/rust-applications.md`` §7): write the + declared files (path-confined) under ``source.project_root``, then hand the plan's + ``toolchain_request`` to ``chain``'s registered + :class:`~composer.rustapp.toolchain.ProjectToolchain`. Returns what the prep established, which + the caller reports back to the wheel as ``AuthorInput.prep_facts`` — empty when the plan only + placed files. + + The split is the seam: writing files is the same in every ecosystem, while preparing a *project* + means driving a build system the host does not understand, which only something that knows the + chain can do (see the toolchain module for why the framework carries no implementation). Either + way the wheel supplies only file contents + a request its chain's toolchain understands, never a + command line, so the network posture stays Python-owned. + + The whole ``source`` goes through rather than just its root: an implementation resolves its own + project facts from it (Solana reads the crate that owns ``relative_path`` to fill in an IDL's + program id), which is knowledge the framework would otherwise have to hold a shape for.""" + workdir = Path(source.project_root) + plan = parse_workspace_prep(module.workspace_prep(input.model_dump_json())) + for rel, contents in plan.files.items(): + target = confined_target(workdir, rel) + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text(contents) + + if not plan.needs_toolchain: + return {} + facts = await project_toolchain(chain).prepare( + plan, input, source=source, sandbox=sandbox, timeout_s=command_timeout_s + ) + if facts: + _log.info("workspace prep established %s", facts) + return facts + + +class PreflightFailed(RuntimeError): + """The prepared workspace does not build, or its skeleton artifact does not run — established + before any property or authored artifact exists. + + Terminal by construction: what fails here is the *workspace* (a dependency graph that won't + resolve, a unit that won't link, codegen the generator rejects, a built program that won't load), + and none of that is something an authoring agent can fix — it doesn't own the project's build + files. Re-authoring against it only burns the revise budget on errors the model can't address, + which is exactly what this gate exists to prevent.""" + + +async def run_preflight_gate( + module: RustAppModule, + input: AuthorInput, + *, + workdir: Path, + sandbox_dict: BackendSpec, +) -> None: + """Gate the prepared workspace with a ``kind="preflight"`` ``compile`` — the wheel's own + skeleton artifact, built by the real toolchain under the real sandbox. + + There is no ``spec`` at all — ``None``, not an empty one: nothing has been authored yet (this + runs alongside system analysis), so the wheel renders the smallest artifact that still exercises + what an authored one will depend on, and a wheel whose toolchain would take an empty spec file + for a real one can tell the two apart. Raises :class:`PreflightFailed` with the compiler + diagnostics the wheel extracted; there is no retry. + + Nothing is streamed as it goes: this is not a graph run, so there is no stream writer to emit + on, and the one thing worth showing — the diagnostics — is what the exception carries.""" + result = parse_compile( + await asyncio.to_thread( + module.compile, input.model_dump_json(), None, str(workdir), json.dumps(sandbox_dict) + ) + ) + if isinstance(result, CompileOk): + return + raise PreflightFailed( + "the prepared workspace does not build (or its skeleton does not run), before anything has " + "been authored — a toolchain, dependency or program-build problem, not something the run " + f"can author its way around:\n{result.errors}" + ) + + +# --------------------------------------------------------------------------- +# The formalizer. +# --------------------------------------------------------------------------- + +class RustFormalizer(Formalizer[RustFormalResult, FeatureUnit]): + """Drives a Rust :class:`~autoprover_sdk.Backend` through one authoring session per unit. + Ecosystem-agnostic: the unit is any :class:`FeatureUnit`, marshalled via ``feature_json()``.""" + + def __init__( + self, + module: RustAppModule, + descriptor: AppDescriptor, + *, + sandbox: SandboxConfig | None = None, + command_timeout_s: int = DEFAULT_TIMEOUT_S, + command_sem: asyncio.Semaphore | None = None, + declared_args: dict[str, Any] | None = None, + setup_result: str | None = None, + project: "ProjectFacts | None" = None, + ): + super().__init__(RustFormalResult, descriptor.backend_tag) + self._module = module + self._descriptor = descriptor + self._sandbox = sandbox + self._command_timeout_s = command_timeout_s + self._command_sem = command_sem + # The run's values for the wheel's own declared flags, put on every component's input. + self._declared_args = declared_args or {} + # The compiled setup spec (Crucible's fixture): on every component's input, and forwarded + # to ``finalize`` so a callout-mode wheel can render the whole deliverable. A wheel that + # declares a ``setup`` step reaches here only through :class:`RustStagedFormalizer`, which + # authors the artifact before constructing this. + self._setup_result = setup_result + # What the preflight established about the project (see :class:`ProjectFacts`), carried on + # every ``AuthorInput`` and mirrored into ``finalize`` — what ships must name the same + # dependency the gated builds did. + self._project = project or ProjectFacts() + + async def _sandbox_spec(self, workdir: Path) -> BackendSpec: + if self._sandbox is None or not self._sandbox.enabled: + return {"argv_prefix": [], "timeout_s": self._command_timeout_s} + return await self._sandbox.backend_spec(workdir, timeout_s=self._command_timeout_s) + + # -- the session ------------------------------------------------------- + + @override + async def formalize( + self, + label: str, + feat: FeatureUnit, + props: list[PropertyFormulation], + ctx: WorkflowContext[RustFormalResult], + run: PipelineRun, + extra_tools: ToolBinder[Any] + ) -> RustFormalResult | GaveUp: + workdir = Path(run.source.project_root) + input = ComponentInput( + program=str(run.source.contract_name), + source_unit=self._project.source_unit, + unit=feat.feature_json(), + props=_properties([UnitProperty(feat.display_name, p) for p in props]), + setup=self._setup_result, + prep_facts=self._project.prep_facts, + args=self._declared_args, + ) + outcome = await run_session( + module=self._module, + input=input, + kind="component", + titles=[p.title for p in props], + env=run.env, + ctx=ctx, + run=run, + workdir=workdir, + sandbox_dict=await self._sandbox_spec(workdir), + descriptor=self._descriptor, + emit=emit_event, + command_sem=self._command_sem, + description=label, + ) + if isinstance(outcome, GaveUp): + return outcome + return RustFormalResult( + commentary=outcome.commentary, + artifact_text=outcome.spec, + checks=outcome.property_checks, + skipped=outcome.skipped, + verdicts=outcome.verdicts, + # What the stamping run actually covered, in the order the host ran it — each target + # with its checks. A callout-mode wheel keys its deliverable sections on the names, and + # carrying the checks alongside is what makes "which properties are these results + # about?" answerable even for a target that errored as a whole. + targets=outcome.ran, + ) + + @override + async def fetch_verdicts( + self, formalized: Formalized[RustFormalResult] + ) -> dict[RuleName, Verdict]: + return { + name: Verdict( + outcome=v.outcome, + line=v.line, + duration_seconds=v.duration_seconds, + unit_file=v.unit_file or formalized.unit_file, + message=v.detail, + ) + for name, v in formalized.result.verdicts.items() + } + + @override + async def finalize( + self, outcomes: list[ComponentOutcome[RustFormalResult, FeatureUnit]], run: PipelineRun + ) -> None: + components = [ + FinalizeComponent(name=o.feat.display_name, outcome=ComponentGaveUp()) + if not isinstance(o.result, Delivered) + # A callout-mode wheel renders the whole deliverable from these (Crucible: folds each + # section into the shared crate, keyed by its property_checks feature) — including the + # targets each check ran under, which its sections and declared features key on. + else FinalizeComponent( + name=o.feat.display_name, + outcome=WireDelivered( + unit_file=o.result.unit_file, + run_link=o.result.run_link, + artifact_text=o.result.result.artifact_text, + property_checks=o.result.result.property_checks(), + skipped=[ + WireSkipped(property_title=sk.property_title, reason=sk.reason) + for sk in o.result.result.skipped + ], + targets=[t.name for t in o.result.result.targets], + ), + ) + for o in outcomes + ] + payload = FinalizeInput( + program=str(run.source.contract_name), + source_unit=self._project.source_unit, + prep_facts=self._project.prep_facts, + components=components, + setup=self._setup_result, + ) + raw = await asyncio.to_thread(self._module.finalize, payload.model_dump_json()) + if not raw: + return + files = parse_files(raw) + root = Path(run.source.project_root) + for rel, contents in files.items(): + target = confined_target(root, rel) + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text(contents) + + +@dataclass(frozen=True) +class ProjectFacts: + """What the host established about the project under analysis — the outcome of + :meth:`RustBackend.preflight`, handed forward to ``prepare_system`` and carried on every callout + from there. + + Both fields are inputs every later callout needs, and neither follows from the analyzed model. + Both are also **chain-shaped**: this is the one part of the seam whose vocabulary belongs to the + analyzed project's build system rather than to the framework, so the host transports them without + a schema (see :mod:`composer.rustapp.toolchain`). Carried rather than recomputed so the gated + preflight build, every authoring turn, and the delivered artifact all agree on what they are + building against.""" + + #: Where the analyzed source lives as a unit of its own build system, from the chain's toolchain + #: (:func:`source_unit_of`). Empty = nothing resolved, and the wheel applies its own convention. + source_unit: dict[str, Any] = field(default_factory=dict) + #: What the workspace prep established (:func:`run_workspace_prep`). Empty = it established + #: nothing, which is what the wheel reads to decide how it sources the program's types. + prep_facts: dict[str, Any] = field(default_factory=dict) + + +# Authors the shared setup spec for a run, given the properties it must make checkable. Built by +# :class:`RustPreparedSystem` and called from :meth:`RustStagedFormalizer.begin` — see there for why +# it runs between extraction and the per-unit fan-out rather than during prep or on first use. +type SetupAuthor = Callable[ + [list[UnitProperty], Sequence[FeatureUnit], PipelineRun], Awaitable[str] +] + + +class RustStagedFormalizer(StagedFormalizer[RustFormalResult, FeatureUnit]): + """The formalizer for a wheel that declares a ``setup`` step, before its shared spec exists. + + ``author`` writes and compiles the artifact from the properties it must make checkable; + ``build`` turns that artifact into the :class:`RustFormalizer` (see + :meth:`RustPreparedSystem.prepare_formalization`, which closes over everything else the + formalizer needs). Splitting it this way means the artifact is never assigned onto a live + formalizer — the only formalizer that exists already has it.""" + + def __init__(self, author: SetupAuthor, build: Callable[[str], RustFormalizer]): + self._author = author + self._build = build + + @override + async def begin( + self, jobs: Sequence[BackendJob[FeatureUnit]], run: PipelineRun + ) -> RustFormalizer: + """Author the shared setup spec from **every** unit's properties, and hand back the + formalizer built around it. + + Two constraints fix this point in the run. It cannot happen in ``prepare_formalization`` + (which overlaps property extraction, so no properties exist yet), and it cannot happen + lazily on first ``formalize`` (whichever unit won the race would decide the artifact the + rest are then told to work within — see :class:`StagedFormalizer` and + docs/crucible-component-units.md (PR3) §8.2). The driver calls this exactly between the two. + + It is also the only moment the whole **unit set** is known, so that goes to the author too: + scaffolding for a multi-unit build is a function of the set rather than of any one unit, and + a wheel whose setup gate builds it needs the set to build the real thing.""" + union = [UnitProperty(job.feat.display_name, prop) for job in jobs for prop in job.props] + return self._build(await self._author(union, [job.feat for job in jobs], run)) + + +@dataclass +class RustPreparedSystem(PreparedSystem[RustFormalResult, FeatureUnit, Any]): + """Generic prepared system, descriptor-driven: author the optional shared ``setup`` artifact and + build a formalizer carrying the injected context. + + The workspace itself was prepared and gated *before* analysis, by + :meth:`RustBackend.preflight` — its outcome arrives here as :attr:`preflight`. + + Descriptor-driven throughout (``docs/rust-applications.md``), so an application needing a shared + fixture, per-run serialization, or a context-thread of the fixture + declared args declares them + rather than subclassing.""" + + backend: "RustBackend" + preflight: ProjectFacts + analyzed: BaseApplication | None = None + + @override + async def prepare_formalization( + self, run: PipelineRun + ) -> Formalizer[RustFormalResult, FeatureUnit] | StagedFormalizer[RustFormalResult, FeatureUnit]: + """A wheel that declares no ``setup`` step gets its formalizer here. One that does gets a + :class:`RustStagedFormalizer` instead — this method overlaps property extraction, so the + properties its artifact must be authored from do not exist yet.""" + b = self.backend + descriptor = b.descriptor + workdir = Path(run.source.project_root) + program = str(run.source.contract_name) + # One shared workspace / build dir → serialize the toolchain runs (declared by the wheel). + command_sem = asyncio.Semaphore(1) if descriptor.serialize_toolchain else None + + analyzed_json = self.analyzed.model_dump(mode="json") if self.analyzed is not None else {} + project = self.preflight + + def build(setup_result: str | None) -> RustFormalizer: + """The formalizer, around a shared setup spec that is either already authored or + not called for.""" + return RustFormalizer( + b.module, b.descriptor, sandbox=b.sandbox, + command_timeout_s=b.command_timeout_s, + command_sem=command_sem, declared_args=b.declared_args, + setup_result=setup_result, project=project, + ) + + setup = descriptor.step(PhaseRole.SETUP) + if setup is None: + return build(None) + # The base for the setup spec's own input; ``author_setup`` adds the properties. + prep_input = SetupInput( + program=program, source_unit=project.source_unit, model=analyzed_json, + prep_facts=project.prep_facts, args=b.declared_args, + ) + + async def author_setup( + props: list[UnitProperty], units: Sequence[FeatureUnit], run: PipelineRun + ) -> str: + # The properties are what the artifact must make checkable, so they are part of both + # the prompt and the cache identity + setup_input = prep_input.with_props(_properties(props)) + # Cached like a formalization result (and skipped entirely on a hit): authoring + + # compiling this is a full LLM loop, and on a large program the longest single step + # of a run — so a re-run after a failure downstream must not pay for it twice. Keyed + # by what it is authored *from*, so a changed model, program crate, type source + # (crate vs IDL) or property set re-authors it. As with the driver's other caches, a + # change to the *prompt* does not invalidate — clear the namespace for that. + setup_ctx = run.ctx.child(RUST_SETUP_KEY(descriptor.name, setup_input)) + if (hit := await setup_ctx.cache_get(RustSetupSpec)) is not None: + return hit.source + # Attached *after* the identity, deliberately: the unit set is what a wheel's gate builds + # the crate's scaffolding from, not something the artifact is authored from, so a changed + # slug must not throw away an artifact that is still correct. + setup_input = setup_input.with_units([u.feature_json() for u in units]) + sandbox_dict = await b.sandbox_spec(workdir) + fixture = await run.runner( + b.task_info(setup), + lambda: run_session( + module=b.module, + input=setup_input, + kind="setup", + titles=[o.prop.title for o in props], + env=run.env, + ctx=setup_ctx, + run=run, + workdir=workdir, + sandbox_dict=sandbox_dict, + descriptor=descriptor, + emit=emit_event, + command_sem=command_sem, + description=setup.label, + ), + ) + if isinstance(fixture, GaveUp): + raise RuntimeError(f"{descriptor.name} setup gave up: {fixture.reason}") + await setup_ctx.cache_put(RustSetupSpec(source=fixture.spec)) + return fixture.spec + + return RustStagedFormalizer(author_setup, build) + + +@dataclass +class RustBackend: + """A ``PipelineBackend`` (satisfied structurally) backed by a Rust wheel. Ecosystem-agnostic: + it locates the main + and marshals units through the resolved ``ecosystem`` + the ``FeatureUnit`` protocol, so its + unit / main / app axes stay open (``Any``) where a single-ecosystem backend would pin them. + + Subclass (or replace via ``backend_cls``) when the app needs non-generic prep — e.g. + Crucible's shared fixture + harness crate.""" + + module: RustAppModule + descriptor: AppDescriptor + #: The application's resolved phase model, held whole so the enum and the core-phase mapping + #: cannot come from two different builds (see :class:`PhaseModel` on why identity matters). + phases: PhaseModel + store: ArtifactStore[Any, RustFormalResult] + ecosystem: Ecosystem[Any, Any, Any] + # Wall-clock ceiling for a single compile/validate (a first build can be minutes). + command_timeout_s: int = DEFAULT_TIMEOUT_S + # How to confine every toolchain run (docs/command-sandbox.md). None → unsandboxed. + sandbox: SandboxConfig | None = None + # Parsed values of the descriptor's declared CLI args, put on every component's + # ``AuthorInput.args`` (e.g. Crucible's ``fuzz_timeout``). Set by the entry point. + declared_args: dict[str, Any] = field(default_factory=dict) + + @property + def phase(self) -> type[enum.Enum]: + """The phase enum synthesized from the descriptor — the *same* class object the frontend's + ``phase_labels`` are keyed by, since the lookup is by member identity. Public because the + prepared system and the formalizer need to tag their own tasks with a declared phase; reach + it through :meth:`task_info` rather than indexing it.""" + return self.phases.phase + + @property + def core_phases(self) -> CorePhases: + return self.phases.core + + # Both are the wheel's to state, so they are derived from its descriptor rather than passed. + @property + def backend_guidance(self) -> str: + return self.descriptor.backend_guidance + + @property + def analysis_spec(self) -> SystemAnalysisSpec: + return SystemAnalysisSpec(self.descriptor.analysis_key, "rust-properties") + + @property + def artifact_store(self) -> ArtifactStore[Any, RustFormalResult]: + return self.store + + def task_info(self, phase: PhaseSpec) -> TaskInfo[enum.Enum]: + """The task a step-declaring phase runs as: an id from its role, the wheel's label, and the + phase *member* itself. + + The one way to turn a declared phase into a member. The enum is synthesized per application, + so a caller can't name a member statically — and resolving it here keeps the member the + driver emits identical to the one the frontend's labels are keyed by.""" + return TaskInfo( + f"{self.descriptor.name}-{phase.role.value}", phase.label, self.phase[phase.key] + ) + + async def preflight(self, run: PipelineRun) -> ProjectFacts: + """Prepare the wheel's workspace and gate it — everything buildable before the program has + been analyzed, run concurrently with system analysis (``docs/rust-applications.md`` §4.2). + + Two steps, both *declared* by the wheel and executed here (``docs/rust-applications.md`` §7): + + 1. :func:`run_workspace_prep` — place the wheel's build files, and (through the chain's + :class:`~composer.rustapp.toolchain.ProjectToolchain`) carry out whatever preparing the + analyzed project takes. Already the run's slowest non-LLM step. + 2. :func:`run_preflight_gate`, when the descriptor declares a ``preflight`` — build a + skeleton artifact *the wheel authors itself* through the real toolchain, in the real + sandbox. This is what turns step 1 from "we placed some build files" into "this workspace + compiles": warming a dependency cache resolves a graph but compiles nothing, and its + failures are deliberately non-fatal. Without the gate the first *check* of the workspace + is the first authored draft's build — after the whole extraction phase, and reported as + compiler errors an authoring agent cannot fix because it does not own the build files. + + Neither step reads the analyzed model or any property, which is what makes the overlap safe. + A failure raises (:class:`PreflightFailed` from the gate, or the workspace toolchain's own + error), and the driver cancels the analysis racing it.""" + descriptor = self.descriptor + gate = descriptor.step(PhaseRole.PREFLIGHT) + workdir = Path(run.source.project_root) + # Resolved once per run and carried on every AuthorInput from here on: the wheel renders its + # build files from this, so prep, every gated build, and the deliverable agree on what they + # are building against. + unit = source_unit_of(self.ecosystem, run.source) + # Declared args are in scope from the start: prep may need one (Crucible reads + # ``program_idl`` when deciding how to source the program's types). + prep_input = PreflightInput( + program=str(run.source.contract_name), + source_unit=unit, args=dict(self.declared_args), + ) + + prep_facts = await run_workspace_prep( + self.module, prep_input, chain=self.ecosystem.name, source=run.source, + sandbox=self.sandbox, command_timeout_s=self.command_timeout_s, + ) + result = ProjectFacts(source_unit=unit, prep_facts=prep_facts) + if gate is None: + return result + + async def gate_workspace() -> None: + await run_preflight_gate( + self.module, + prep_input.with_prep_facts(result.prep_facts), + workdir=workdir, + sandbox_dict=await self.sandbox_spec(workdir), + ) + + # This is a build, not an agent: it belongs to the run's CPU budget, not to the + # ``--max-concurrent`` agent slots it would otherwise hold for the whole of system analysis. + await run.cpu_runner(self.task_info(gate), gate_workspace) + return result + + async def sandbox_spec(self, workdir: Path) -> BackendSpec: + """The confinement prefix the wheel's blocking callouts prepend, or the trusted empty one.""" + if self.sandbox is not None and self.sandbox.enabled: + return await self.sandbox.backend_spec(workdir, timeout_s=self.command_timeout_s) + return {"argv_prefix": [], "timeout_s": self.command_timeout_s} + + async def prepare_system( + self, analyzed: BaseApplication, run: PipelineRun, preflight: ProjectFacts + ) -> PreparedSystem[RustFormalResult, FeatureUnit, Any]: + return RustPreparedSystem( + self.ecosystem.locate_main(analyzed, run.source), self, preflight, analyzed + ) + + def to_artifact_id(self, c: FeatureUnit) -> RustArtifact: + return RustArtifact( + c.slug, + self.descriptor.artifact_layout.artifact_prefix, + self.descriptor.artifact_layout.artifact_extension, + ) diff --git a/composer/rustapp/cli.py b/composer/rustapp/cli.py new file mode 100644 index 00000000..12b17fd4 --- /dev/null +++ b/composer/rustapp/cli.py @@ -0,0 +1,129 @@ +"""Generic ``main()`` glue for a Rust application. + +Two shapes, differing only in who owns the event loop (identical to the built-in +apps' ``composer/cli/*.py``): + +* :func:`tui_main` — the pipeline runs as a background worker inside the Textual + app, streaming into it. +* :func:`console_main` — the pipeline runs directly, printing on completion. + +A Rust application ships a two-line CLI: + + from composer.rustapp.cli import tui_main + def main() -> int: + return tui_main("my_app") + +``import composer.bind`` runs first (import-time DI / test-tape bootstrap), exactly +as the built-in ``main()``s require. +""" + +import asyncio +import logging + +import composer.bind as _ # noqa: F401 (side-effecting DI/tape bootstrap; must load first) + +from composer.diagnostics.timing import RunSummary +from composer.pipeline.core import CorePipelineResult +from composer.rustapp.entry import EnvBuilder, rust_entry_point +from composer.rustapp.frontend import GenericRustApp, GenericRustConsoleHandler +from composer.rustapp.host import RustApplication, build_application +from composer.rustapp.result import RustFormalResult +from composer.rustapp.results import format_verdict_lines, summarize_verdicts + +_log = logging.getLogger(__name__) + + +def _event_kinds(app: RustApplication) -> set[str]: + return {e.kind for e in app.descriptor.event_kinds} + + +def _notice_kinds(app: RustApplication) -> set[str]: + return {e.kind for e in app.descriptor.event_kinds if e.notice} + + +def _verdict_lines( + app: RustApplication, result: CorePipelineResult[RustFormalResult] +) -> list[str]: + """Per-check verdict tally + listing when the results carry verdicts; empty otherwise + (a run-service backend, or a wheel that bakes none).""" + return format_verdict_lines( + summarize_verdicts(result, app.descriptor.backend_tag) + ) + + +async def _tui_main(module_name: str, *, env_builder: EnvBuilder | None = None) -> int: + summary = RunSummary() + app_meta = build_application(module_name) + async with rust_entry_point(app_meta, summary, env_builder=env_builder) as pipeline: + tui = GenericRustApp( + phase_labels=app_meta.phases.labels, + section_order=app_meta.phases.section_order, + header_text=app_meta.header_text, + event_kinds=_event_kinds(app_meta), + notice_kinds=_notice_kinds(app_meta), + ) + result: CorePipelineResult[RustFormalResult] | None = None + + async def work(): + nonlocal result + try: + result = await pipeline(tui.make_handler) + msg = ( + f"{app_meta.name} complete: {result.n_components} " + f"{app_meta.descriptor.unit_noun(plural=True)}, " + f"{result.n_properties} properties" + ) + if tally := summarize_verdicts( + result, app_meta.descriptor.backend_tag + ).tally: + msg += f" — {tally}" + if result.failures: + msg += f", {len(result.failures)} failures" + tui.notify(msg) + except Exception as exc: # noqa: BLE001 — surface to the UI, don't crash the loop + # A toast alone loses the failure the moment it fades — and the traceback with it. + _log.exception("pipeline failed") + tui.notify(f"Pipeline failed: {exc}", severity="error") + finally: + tui.mark_pipeline_done() + + tui.set_work(work) + await tui.run_async() + print(summary.format()) + if result is not None: + for line in _verdict_lines(app_meta, result): + print(line) + for f in result.failures: + print(f" FAILED: {f}") + return 0 + + +async def _console_main(module_name: str, *, env_builder: EnvBuilder | None = None) -> int: + summary = RunSummary() + app_meta = build_application(module_name) + async with rust_entry_point(app_meta, summary, env_builder=env_builder) as run: + result = await run(GenericRustConsoleHandler(_event_kinds(app_meta)).make_handler) + print(f"\n{'=' * 60}") + print(summary.format()) + # The counts-block noun for the formalized units ("Components" / "Instructions"). + units = app_meta.descriptor.unit_noun(plural=True).capitalize() + print(f"\n {units}: {result.n_components}") + print(f" Properties: {result.n_properties}") + for line in _verdict_lines(app_meta, result): + print(line) + if result.failures: + print(f" Failures: {len(result.failures)}") + for f in result.failures: + print(f" - {f}") + print(f"{'=' * 60}") + return 0 + + +def tui_main(module_name: str, *, env_builder: EnvBuilder | None = None) -> int: + """Run ``module_name`` as a Textual TUI application. Blocks until the run ends.""" + return asyncio.run(_tui_main(module_name, env_builder=env_builder)) + + +def console_main(module_name: str, *, env_builder: EnvBuilder | None = None) -> int: + """Run ``module_name`` in console (no-TUI) mode. Blocks until the run ends.""" + return asyncio.run(_console_main(module_name, env_builder=env_builder)) diff --git a/composer/rustapp/descriptor.py b/composer/rustapp/descriptor.py new file mode 100644 index 00000000..274dce7f --- /dev/null +++ b/composer/rustapp/descriptor.py @@ -0,0 +1,241 @@ +"""Python mirror of the Rust ``AppDescriptor`` (see ``autoprover-sdk``). + +These pydantic models are the Python side of the descriptor ABI. They are parsed +from the JSON a Rust wheel returns from ``descriptor()`` and consumed by the host +to synthesize the phase enum, argparse, frontend and artifact store. Keep the +field names in lockstep with ``rust/autoprover-sdk/src/lib.rs``. +""" + +import enum +from typing import Annotated, Literal + +from pydantic import Field + +from composer.rustapp.wire import WireModel + +from composer.spec.source.report.schema import ReportBackend + +#: Ecosystem/chain tag. Mirrors ``composer.pipeline.ecosystem.ChainTag`` (kept local so this +#: ABI-mirror module stays decoupled from the pipeline); the host resolves it against the +#: ecosystem registry. +ChainTag = Literal["evm", "solana", "soroban"] + + +class PhaseRole(str, enum.Enum): + """Which step of the run a declared phase groups — and, for the steps the host runs as their own + visible task (:data:`STEP_ROLES`), the declaration *of* that step. + + The four the *driver* runs are :meth:`required` and every application must claim them; the rest + are optional steps the host runs around it. A role no phase claims is a step this application + does not have; :attr:`GROUPING` is a phase that declares no step at all.""" + + #: Grouping only — the host runs no step of its own here (cf. autoprove's harness/autosetup). + GROUPING = "grouping" + ANALYSIS = "analysis" + EXTRACTION = "extraction" + FORMALIZATION = "formalization" + REPORT = "report" + #: Design-doc discovery, which the *entry point* runs before the pipeline (only when the doc + #: wasn't passed on the command line). Optional: unclaimed, the task is grouped under the first + #: declared phase. A wheel that wants it in a section of its own claims this role rather than + #: relying on a phase key the host would have to recognize by name. + DISCOVERY = "discovery" + #: The analysis-independent gate on the prepared workspace, run concurrently with system + #: analysis. The host follows the wheel's ``workspace_prep`` with a ``preflight`` ``compile`` + #: that carries no ``spec`` at all: the wheel renders its own minimal skeleton, since nothing + #: has been authored yet. + #: + #: It exists to fail on a *toolchain* problem — an unresolvable dependency graph, a harness that + #: doesn't link, IDL codegen the generator rejects — while the run has spent almost no LLM + #: budget. Such a failure is terminal (the host raises rather than re-authoring: the author does + #: not own the manifest and cannot fix it), which lets the driver cancel the analysis and + #: extraction running alongside. + PREFLIGHT = "preflight" + #: A shared setup spec authored once before per-component formalization (Crucible's shared + #: fixture). The host runs the author→compile loop for a :class:`SetupInput + #: ` and hands the compiled spec to every component as + #: ``AuthorInput.setup``. + SETUP = "setup" + + @classmethod + def required(cls) -> tuple["PhaseRole", ...]: + """The roles every application must claim — the four steps the shared driver itself runs, + and therefore tags every run with.""" + return (cls.ANALYSIS, cls.EXTRACTION, cls.FORMALIZATION, cls.REPORT) + + +#: The roles whose step the host runs as its own visible task, ``{app}-{role}``. Looked up through +#: :meth:`AppDescriptor.step` — an unclaimed one means the application has no such step. +STEP_ROLES: tuple[PhaseRole, ...] = (PhaseRole.PREFLIGHT, PhaseRole.SETUP) + + +class PerComponent(WireModel): + """The generic store writes one ``{prefix}_{slug}.{ext}`` file per component.""" + + mode: Literal["per_component"] = "per_component" + + +class Callout(WireModel): + """The store writes no per-component source; the wheel's ``finalize`` renders the whole + deliverable (e.g. Crucible's one shared crate).""" + + mode: Literal["callout"] = "callout" + #: Does the deliverable have a representative primary file? ``finalize`` renders a whole tree, + #: but every delivered component still records one path — its basename becomes the component's + #: ``unit_file``, the report's rule-identity fallback, echoed to ``finalize`` — and the store + #: can't guess where in that tree the components' checks land. A path (project-relative, + #: ``{program}``-templated — Crucible: ``fuzz/{program}/src/main.rs``) names that file; + #: ``None`` declares that no one file represents the deliverable, and components anchor to the + #: layout's ``deliverable_dir`` instead. On the variant because it means nothing per-component. + deliverable_path: str | None + + +#: How the source deliverable is written — tagged on ``mode`` (Rust ``DeliverableMode``). +DeliverableMode = Annotated[PerComponent | Callout, Field(discriminator="mode")] + + +class PhaseSpec(WireModel): + """One task-grouping phase; ``key`` becomes the synthesized enum member name. + + For a :data:`STEP_ROLES` role this is also the declaration of that step: the task the host runs + is this ``label``, under this phase, with id ``{app}-{role}``. Turn one into its task with + :meth:`composer.rustapp.adapter.RustBackend.task_info`, which resolves the phase member.""" + + key: str + label: str + order: int + role: PhaseRole + + +class StrDefault(WireModel): + """A text flag's default; ``None`` when it has none.""" + + kind: Literal["str"] = "str" + value: str | None + + +class IntDefault(WireModel): + """A numeric flag's default; ``None`` when it has none.""" + + kind: Literal["int"] = "int" + value: int | None + + +class BoolDefault(WireModel): + """A ``store_true`` flag's initial state. Not optional the way the other two are: a boolean + flag is either set or unset, so there is no third "no default" case to spell.""" + + kind: Literal["bool"] = "bool" + value: bool + + +#: A declared CLI argument's default — tagged on ``kind`` (Rust ``ArgDefault``). A variant per type +#: rather than one model with a ``str | int | bool | None`` beside the tag: only the matching +#: variant's ``value`` is meaningful, and this way the tag is what narrows it. +ArgDefault = Annotated[StrDefault | IntDefault | BoolDefault, Field(discriminator="kind")] + + +class ArgSpec(WireModel): + """A CLI flag the generic entry point adds beyond the positional inputs.""" + + flag: str + help: str + default: ArgDefault + required: bool + + +class EventKind(WireModel): + """A domain event kind the frontend should render. + + ``notice`` events are surfaced as a persistent, always-visible callout (plus a toast) + rather than a line in the collapsible per-task events log — for one-shot important + results such as one check's verdict. + + The events themselves are emitted by the *host*, around the callouts it drives (a build + failure, a review verdict, a check's outcome) — a wheel has no emit channel, since its blocking + callouts run to completion with the GIL released. A declared kind nothing emits renders + nothing.""" + + kind: str + label: str + notice: bool + + +class ArtifactLayout(WireModel): + """Project-root-relative deliverable layout.""" + + deliverable_dir: str + internal_dir: str + report_dir: str + artifact_dir: str + artifact_prefix: str + artifact_extension: str + property_suffix: str + + +class AppDescriptor(WireModel): + """The complete declaration a Rust wheel exports.""" + + name: str + header_text: str + #: The ecosystem (chain) whose system model / prompts the shared front half uses. The + #: host resolves it against ``composer.pipeline.ecosystem.ECOSYSTEMS``; a tag outside the set + #: fails here, at descriptor load, rather than against the wrong system model mid-run. + ecosystem: ChainTag + #: Which report vocabulary this backend's results are rendered with. Typed (rather than a free + #: ``str`` validated later by ``as_report_backend``) so a wheel declaring a tag the report + #: doesn't know fails in ``model_validate_json`` — at descriptor load, before the run starts — + #: instead of at formalizer construction. The set is closed; see ``ReportBackend``. + backend_tag: ReportBackend + backend_guidance: str + analysis_key: str + phases: list[PhaseSpec] + args: list[ArgSpec] + rag_db_default: str | None + event_kinds: list[EventKind] + artifact_layout: ArtifactLayout + #: How the source deliverable is written (see :data:`DeliverableMode`). + deliverable_mode: DeliverableMode + #: Serialize the blocking toolchain callouts on one semaphore — set when the app shares a + #: single build dir / target across components. + serialize_toolchain: bool + #: Default to the fail-closed ``launcher`` sandbox provider (still overridable by + #: ``COMPOSER_SANDBOX_PROVIDER``). Set by any wheel that runs untrusted native toolchains. + confine_by_default: bool + #: Human noun for one formalized component in the console/TUI summary ("instruction" for + #: Crucible). ``None`` → "component"; read it through :meth:`unit_noun`. + component_noun: str | None + #: What this backend calls one check *to the model* ("rule", "harness function", "invariant") — + #: the word the authoring prompts use throughout. ``None`` → "check"; read it through + #: :meth:`check_label`. Declared rather than fixed because an author writes better when the + #: prompt speaks its domain's language. + check_noun: str | None + #: What an author may cite when rebutting the judge's prior-round feedback — the closed set the + #: rebuttal tool's ``evidence_type`` is built from. Declared per wheel because the evidence a + #: backend can produce is a property of that backend. + evidence_kinds: list[str] + + def unit_noun(self, *, plural: bool = False) -> str: + """The noun for a formalized unit, with the generic default applied — so no frontend + spells the ``or "component"`` fallback (nor the pluralization) itself.""" + noun = self.component_noun or "component" + return f"{noun}s" if plural else noun + + def check_label(self, *, plural: bool = False) -> str: + """The noun for one check, with the generic default applied — so no prompt spells the + ``or "check"`` fallback (nor the pluralization) itself.""" + noun = self.check_noun or "check" + return f"{noun}s" if plural else noun + + def ordered_phases(self) -> list[PhaseSpec]: + return sorted(self.phases, key=lambda p: (p.order, p.key)) + + def role_map(self) -> dict[PhaseRole, str]: + """The declared phase ``key`` for each role a phase claims.""" + return {p.role: p.key for p in self.phases if p.role is not PhaseRole.GROUPING} + + def step(self, role: PhaseRole) -> PhaseSpec | None: + """The phase declaring ``role``'s step, or ``None`` when no phase claims it — which is how + an application says it has no such step. The lookup is by role rather than by a key one + side spells and the other has to match.""" + return next((p for p in self.phases if p.role is role), None) diff --git a/composer/rustapp/entry.py b/composer/rustapp/entry.py new file mode 100644 index 00000000..939e0575 --- /dev/null +++ b/composer/rustapp/entry.py @@ -0,0 +1,408 @@ +"""Generic async entry point for a Rust application. + +Mirrors ``composer/foundry/entry.py``'s shape — parse args → open DB / store / +checkpointer / logging → yield a closure the caller drives with a handler factory +— but is *descriptor-driven*: the CLI flags, precondition validation, and report +tag all come from the Rust wheel's ``AppDescriptor`` instead of being hard-coded. + +The imperative service wiring (Postgres pools, the async tool context, the thread +logger, ``WorkflowContext``) stays Python and is essentially identical to the +foundry entry point — that shell is irreducibly async and is not something Rust +owns (see ``docs/rust-applications.md`` §10). What Rust contributes here is only +declarative: the arg schema and the ``validate_preconditions`` hook. + +The env built here is descriptor-driven too: the standard source-navigation toolset +(``code_explorer`` + fs tools), plus the search tools of the corpus the descriptor +names in ``rag_db_default`` (none by default). A backend that wants a different tool +surface entirely can supply its own builder via ``env_builder=``. +""" + +import argparse +import asyncio +import enum +import json +import os +import pathlib +import sys +import uuid +from contextlib import asynccontextmanager +from functools import partial +from typing import Any, AsyncIterator, Awaitable, Callable, cast + +from langchain_core.tools import BaseTool +from langgraph.store.base import BaseStore + + +from composer.core.user import user_data_ns +from composer.diagnostics.logging_setup import setup_autoprove_logging +from composer.diagnostics.timing import RunSummary, install_run_summary +from composer.input.parsing import add_protocol_args +from composer.input.types import ( + DEFAULT_RECURSION_LIMIT, + ExtendedModelOptions, + TieredModelOptions, +) +from composer.io.multi_job import HandlerFactory, TaskInfo, run_task +from composer.io.thread_logging import default_logging_ns, thread_logger +from composer.pipeline.cli import root_cache_key +from composer.pipeline.core import CorePipelineResult, DEFAULT_MAX_CPU_TASKS +from composer.pipeline.ecosystem import Ecosystem +from composer.rag.models import DefaultEmbedder, get_model +from composer.rustapp.adapter import source_unit_of +from composer.rustapp.descriptor import ArgSpec, BoolDefault, IntDefault, PhaseRole, StrDefault +from composer.rustapp.wire import AppArgs, parse_sandbox_grants +from composer.rustapp.host import RustApplication, build_application, run_application +from composer.rustapp.result import RustFormalResult +from composer.sandbox.config import SandboxConfig +from composer.sandbox.recipes import DEFAULT_ENV_PASSTHROUGH +from composer.spec.context import SourceCode, SourceFields, WorkflowContext +from composer.spec.service_host import ModelProvider, PureServiceHost, ServiceHost +from composer.spec.source.design_doc_finder import ( + DESIGN_DOC_DISCOVERY_TASK_ID, + discovery_cache_key, + resolve_design_doc, +) +from composer.spec.source.source_env import ( + build_basic_source_tools, + build_source_tools, +) +from composer.spec.types import SourceIdentifier +from composer.tools.rag_env import build_rag_tools +from composer.ui.tool_display import async_tool_context +from composer.workflow.services import standard_connections +from composer.llm.registry import get_provider_for + +# A caller-supplied env builder, for backends that want a custom tool/RAG surface. +EnvBuilder = Callable[..., ServiceHost] + +# The Executor a frontend drives. +RustRunner = Callable[ + [HandlerFactory], Awaitable[CorePipelineResult[RustFormalResult]] +] + + +def build_default_env( + *, + model_provider: ModelProvider, + project_root: str, + store: BaseStore, + source_question_ns: tuple[str, ...], + recursion_limit: int, + ecosystem: Ecosystem[Any, Any, Any], + rag_db: str | None = None, +) -> ServiceHost: + """The env for a wheel that supplies no ``env_builder``: the same ``code_explorer`` + fs tools + the built-in backends use for analysis/authoring, plus — when the descriptor declares a + ``rag_db_default`` and it is passed here as ``rag_db`` — that corpus's RAG search tools. + No ``rag_db`` means no RAG surface. + + ``ecosystem`` supplies both the fs-exclusion default (Cargo layout for Rust, Foundry for EVM) + and the code_explorer's prompt.""" + basic = build_basic_source_tools( + root=project_root, forbidden_read=ecosystem.language.default_forbidden_read + ) + full = build_source_tools( + basic, + model_provider, + store, + source_question_ns, + recursion_limit=recursion_limit, + ecosystem=ecosystem, + ) + rag_tools: tuple[BaseTool, ...] = build_rag_tools(rag_db) if rag_db else () + return PureServiceHost( + models=model_provider, rag_tools=rag_tools, sort="existing" + ).bind_source_tools(full) + + +def _build_confinement(app: RustApplication, args: AppArgs) -> SandboxConfig: + """The default command-sandbox config for a wheel that sets ``confine_by_default`` — the + fail-closed ``launcher`` provider (overridable by ``COMPOSER_SANDBOX_PROVIDER``), with the + wheel's ``sandbox_grants`` (extra read-only paths / env names) unioned in. Python owns the + policy; the wheel only *declares* the grants (``docs/rust-applications.md`` §8).""" + grants = parse_sandbox_grants(app.module.sandbox_grants(args.model_dump_json())) + extra_ro = tuple(pathlib.Path(p) for p in grants.extra_ro) + extra_env = tuple(grants.extra_env) + provider = os.environ.get("COMPOSER_SANDBOX_PROVIDER", "launcher") + return SandboxConfig( + provider=provider, + extra_ro=extra_ro, + env_passthrough=DEFAULT_ENV_PASSTHROUGH + extra_env, + ) + + +def _user_ns(*parts: str | None) -> tuple[str, ...]: + return user_data_ns() + tuple(p for p in parts if p) + + +def _root_cache_key( + project_root: str, system_doc_path: pathlib.Path | None, relative_path: str, contract_name: str +) -> str: + return root_cache_key( + project_root=project_root, + system_doc_path=system_doc_path, + relative_path=relative_path, + contract_name=contract_name, + ) + + +def _arg_dest(spec: ArgSpec) -> str: + """The argparse dest a declared flag lands under (``--echo-tag`` → ``echo_tag``).""" + return spec.flag.lstrip("-").replace("-", "_") + + +def _declared_args(args: argparse.Namespace, specs: list[ArgSpec]) -> dict[str, Any]: + """The parsed values of the descriptor's declared flags, keyed by dest — what the host threads + into ``validate_preconditions`` and every component's ``AuthorInput.context``.""" + return {d: getattr(args, d) for d in (_arg_dest(s) for s in specs)} + + +def _add_declared_args(parser: argparse.ArgumentParser, specs: list[ArgSpec]) -> None: + """Add the descriptor's declared flags to ``parser``.""" + for spec in specs: + dest = _arg_dest(spec) + match spec.default: + case BoolDefault() as d: + parser.add_argument( + spec.flag, dest=dest, action="store_true", + default=d.value, help=spec.help, + ) + case IntDefault() as d: + parser.add_argument( + spec.flag, dest=dest, type=int, default=d.value, + required=spec.required, help=spec.help, + ) + case StrDefault() as d: + parser.add_argument( + spec.flag, dest=dest, type=str, default=d.value, + required=spec.required, help=spec.help, + ) + + +def build_arg_parser(app: RustApplication) -> argparse.ArgumentParser: + """The descriptor-driven argument parser — one definition, used by :func:`rust_entry_point` + and exposed for tests / ``--help`` introspection without opening any service.""" + parser = argparse.ArgumentParser( + description=f"{app.descriptor.name} — AutoProver (Rust backend)" + ) + add_protocol_args(parser, ExtendedModelOptions) + parser.add_argument( + "--recursion-limit", type=int, default=DEFAULT_RECURSION_LIMIT, + help=f"Max graph iterations (default: {DEFAULT_RECURSION_LIMIT})", + ) + parser.add_argument("project_root", help="Project root") + parser.add_argument("main_contract", help="Main contract as path:ContractName") + parser.add_argument( + "system_doc", nargs="?", default=None, + help="Path to the design document (text or PDF); auto-discovered if omitted", + ) + parser.add_argument("--max-concurrent", type=int, default=4, help="Max concurrent agents (default: 4)") + parser.add_argument("--max-cpu-tasks", type=int, default=DEFAULT_MAX_CPU_TASKS, help=f"Max concurrent CPU-bound tasks — toolchain builds and the like (default: {DEFAULT_MAX_CPU_TASKS})") + parser.add_argument("--cache-ns", default=None, help="Cache namespace (enables cross-run caching)") + parser.add_argument("--memory-ns", default=None, help="Memory namespace (default: thread id)") + parser.add_argument("--interactive", action="store_true", help="Interactively refine extracted properties") + parser.add_argument("--max-bug-rounds", type=int, default=3, help="Max bug-extraction rounds per component (default: 3)") + _add_declared_args(parser, app.descriptor.args) + return parser + + +def _discovery_phase(app: RustApplication) -> enum.Enum: + """The phase to tag the design-doc-discovery task with: the phase claiming + :attr:`PhaseRole.DISCOVERY`, else the first ordered phase (so a wheel that doesn't care still + groups it somewhere sensible). + + Claimed by slot rather than by a phase *key* the host recognizes: the descriptor already has a + mechanism for "this declared phase fills that role", and a magic key would be a convention a + wheel author has to know to spell exactly right — with no error if they didn't.""" + return app.phases.role_member(PhaseRole.DISCOVERY) or app.phases.first_member + + +@asynccontextmanager +async def rust_entry_point( + app: RustApplication, + summary: RunSummary, + *, + argv: list[str] | None = None, + env_builder: EnvBuilder | None = None, +) -> AsyncIterator[RustRunner]: + """Parse args, open services, and yield the Executor for ``app``. + + Pass a pre-built :class:`RustApplication` (from :func:`build_application`) so the + backend and the frontend share one phase enum. ``argv`` overrides ``sys.argv`` + (useful in tests); ``env_builder`` overrides :func:`build_default_env`.""" + descriptor = app.descriptor + parser = build_arg_parser(app) + args = parser.parse_args(argv) + + project_root = pathlib.Path(args.project_root).resolve() + main_path, contract_name = args.main_contract.split(":", 1) + contract_name = SourceIdentifier(contract_name) + full_path = pathlib.Path(main_path).resolve() + if not full_path.is_relative_to(project_root): + parser.error(f"Invalid path: {full_path} not under project root {project_root}") + relative_path = str(full_path.relative_to(project_root)) + + # The ecosystem's fs-exclusion default (Cargo layout for Rust, Foundry for EVM). The design-doc + # finder works off these source fields alone, and so does the crate resolution below. + forbidden_read = app.ecosystem.language.default_forbidden_read + init_source = SourceFields( + project_root=str(project_root), + contract_name=contract_name, + relative_path=relative_path, + forbidden_read=forbidden_read, + ) + + # The run's inputs as both argument-shaped callouts see them, resolved once here: the wheel + # gets each part as a field rather than re-deriving any of them (``source_unit`` is the same + # blob every ``AuthorInput`` carries, so a wheel can check up-front that the code it will + # depend on is where the host says it is — Crucible: the program crate must exist). + declared_args = _declared_args(args, descriptor.args) + app_args = AppArgs( + project_root=str(project_root), + program=str(contract_name), + source_path=relative_path, + system_doc=args.system_doc or None, + source_unit=source_unit_of(app.ecosystem, init_source), + declared=declared_args, + ) + + # Rust-owned precondition validation (cf. foundry's foundry.toml check). + err = app.validate_preconditions(app_args) + if err: + parser.error(err) + + model = get_model() + + thread_id = f"{descriptor.name}_{uuid.uuid4().hex[:12]}" + text_log, events_log = setup_autoprove_logging(str(project_root), thread_id) + print(f"{descriptor.name} logs: {text_log}\n events: {events_log}", file=sys.stderr) + install_run_summary(summary) + + # argparse Namespace duck-types the protocol: the model flags come from ExtendedModelOptions. + tiered = get_provider_for(tiered=cast(TieredModelOptions, args)) + discovery_phase = _discovery_phase(app) + + async with ( + standard_connections(provider=tiered.provider_service, embedder=DefaultEmbedder(model)) as conns, + async_tool_context(), + thread_logger( + conns.store, + { + "root_thread_id": thread_id, + "workflow": descriptor.name, + "memory_ns": args.memory_ns if args.memory_ns is not None else thread_id, + }, + default_logging_ns(uid=None), + run_id=summary.run_id, + ), + ): + model_provider = ModelProvider( + heavy_model=tiered.heavy, + lite_model=tiered.lite, + checkpointer=conns.checkpointer, + ) + # The finder's cache namespace is doc-independent (keyed by project/contract). + disc_cache_ns: tuple[str, ...] | None = ( + _user_ns( + args.cache_ns, "discovery", + discovery_cache_key(str(project_root), relative_path, str(contract_name)), + ) + if args.cache_ns is not None + else None + ) + disc_ctx = WorkflowContext.create( + services=conns.memory, thread_id=thread_id, store=conns.store, + recursion_limit=args.recursion_limit, memory_namespace=args.memory_ns, + cache_namespace=disc_cache_ns, + ) + semaphore = asyncio.Semaphore(args.max_concurrent) + + async def runner(handler: HandlerFactory) -> CorePipelineResult[RustFormalResult]: + # 1. Resolve the design doc: use the supplied path, else discover one as a + # visible task (needs the handler scope, which only exists here). Discovery + # may come up empty, which is not fatal — see below. + sys_path: pathlib.Path | None + if args.system_doc is not None: + sys_path = pathlib.Path(args.system_doc) + else: + sys_path = await run_task( + factory=handler, + info=TaskInfo( + task_id=DESIGN_DOC_DISCOVERY_TASK_ID, + label="Design Doc Discovery", + phase=discovery_phase, + ), + fn=lambda: resolve_design_doc( + source=init_source, uploader=conns.uploader, + models=model_provider, disc_ctx=disc_ctx, + ), + semaphore=semaphore, + ) + + # ``sys_path`` is None only when discovery found nothing: run source-only. + if sys_path is not None: + content = await conns.uploader.get_document(sys_path) + if content is None: + raise ValueError(f"cannot read design document: {sys_path}") + else: + content = None + + # 2. Doc-dependent construction. The root cache key hashes the doc bytes, so + # a discovered doc and a supplied one produce an identical key. + root_key = _root_cache_key( + str(project_root), sys_path, relative_path, str(contract_name) + ) + cache_root: tuple[str, ...] | None = ( + _user_ns(args.cache_ns, root_key) if args.cache_ns is not None else None + ) + source_input = SourceCode( + content=content, + project_root=str(project_root), + contract_name=contract_name, + relative_path=relative_path, + forbidden_read=forbidden_read, + ) + source_question_ns = _user_ns("source_agent", "cache", root_key) + # Descriptor-driven env: source tools, plus (if the wheel declares a RAG corpus) that + # corpus's search tools. A wheel can still force its own via ``env_builder=``, which + # takes no ``rag_db`` — a custom builder owns its whole tool surface. + builder = env_builder or partial( + build_default_env, rag_db=app.descriptor.rag_db_default + ) + env = builder( + model_provider=model_provider, + project_root=str(project_root), + store=conns.indexed_store, + source_question_ns=source_question_ns, + recursion_limit=args.recursion_limit, + ecosystem=app.ecosystem, + ) + ctx = WorkflowContext.create( + services=conns.memory, + thread_id=thread_id, + store=conns.store, + recursion_limit=args.recursion_limit, + cache_namespace=cache_root, + memory_namespace=args.memory_ns, + ) + + # Thread the declared CLI args into the backend (→ every component's context) and, + # if the wheel asks to be confined by default, build the launcher policy with its + # declared sandbox grants. Both are inert for a wheel that declares neither. + app.options.declared_args = declared_args + if app.options.sandbox is None and app.descriptor.confine_by_default: + app.options.sandbox = _build_confinement(app, app_args) + + return await run_application( + app, + source_input=source_input, + ctx=ctx, + handler_factory=handler, + env=env, + max_concurrent=args.max_concurrent, + max_cpu_tasks=args.max_cpu_tasks, + max_bug_rounds=args.max_bug_rounds, + interactive=args.interactive, + ) + + yield runner diff --git a/composer/rustapp/frontend.py b/composer/rustapp/frontend.py new file mode 100644 index 00000000..20e8ef0c --- /dev/null +++ b/composer/rustapp/frontend.py @@ -0,0 +1,150 @@ +"""Generic frontend for a Rust application. + +Both frontends are thin, descriptor-driven subclasses of the shared bases: + +* :class:`GenericRustApp` — a ``MultiJobApp`` TUI whose phase labels / section + order come from the descriptor. +* :class:`GenericRustConsoleHandler` — the stdout ``HandlerFactory``. + +Domain-event rendering is *data-driven* by the descriptor's ``event_kinds``: a +Rust ``Command::Emit`` becomes a ``{"type": kind, ...}`` custom-stream payload, +which the handler writes to the task's log if ``kind`` is a declared event kind. +No per-application Python subclass is needed — the same generic handler renders +any Rust app's events (see ``docs/rust-applications.md`` §10). +""" + +import json +from collections.abc import Set as AbstractSet +from typing import Any, override + +from rich.text import Text +from textual.containers import VerticalScroll +from textual.widgets import Collapsible, RichLog + +from composer.io.event_handler import EventHandler, NullEventHandler +from composer.io.multi_job import TaskInfo +from composer.ui.multi_console_handler import MultiJobConsoleHandler +from composer.spec.source.report.render import outcome_glyph +from composer.spec.source.report.schema import Outcome +from composer.ui.multi_job_app import MultiJobApp, MultiJobTaskHandler, TaskHost +from composer.ui.tool_display import ToolDisplayConfig + + +def _render_event(payload: dict) -> str: + """A one-line rendering of an emit payload: prefer a ``line`` field, else a + compact JSON of everything but the discriminating ``type``.""" + if isinstance(payload.get("line"), str): + return payload["line"] + rest = {k: v for k, v in payload.items() if k != "type"} + return json.dumps(rest) if rest else "" + + +def _notice_headline(payload: dict) -> str: + """The persistent-callout headline for a notice event: its one-line rendering, prefixed with an + outcome glyph when the payload names one the report knows. + + The glyph table is the report's (``outcome_glyph``), not a copy: a ✓ has to mean the same thing + here, in the console rollup and in the HTML. An ``outcome`` this host doesn't recognize just + goes unmarked.""" + body = _render_event(payload) + raw = payload.get("outcome") + outcome = Outcome.parse(raw) if isinstance(raw, str) else None + return f"{outcome_glyph(outcome)} {body}" if outcome is not None else body + + +class GenericRustTaskHandler(MultiJobTaskHandler[None], NullEventHandler): + """Per-task handler that streams the app's declared domain events into a + collapsible log pinned at the top of the task panel.""" + + def __init__( + self, + task_id: str, + label: str, + panel: VerticalScroll, + host: TaskHost, + tool_config: ToolDisplayConfig, + event_kinds: set[str], + notice_kinds: AbstractSet[str] = frozenset(), + ): + super().__init__(task_id, label, panel, host, tool_config) + self._event_kinds = event_kinds + # Notice kinds are surfaced as a persistent callout (post_notice) instead of a + # buried log line; ``event_kinds`` here is the streaming (non-notice) remainder. + self._notice_kinds = notice_kinds + self._event_log: RichLog | None = None + + def format_hitl_prompt(self, ty: None) -> list[Text | str]: + raise NotImplementedError("Rust applications do not use HITL interrupts") + + async def _ensure_event_log(self) -> RichLog: + if self._event_log is None: + log = RichLog(highlight=True, markup=False) + log.styles.min_height = 12 + self._event_log = log + # Created lazily so an event-less task — e.g. design-doc discovery — + # never grows an empty Events section. + await self._mount_fixture(Collapsible(log, title="Events")) + return self._event_log + + @override + async def handle_event(self, payload: dict, path: list[str], checkpoint_id: str) -> None: + kind = payload.get("type") + if kind in self._notice_kinds: + # A one-shot important result — a persistent callout in the panel + a toast, + # visible without expanding the events log. + await self.post_notice(_notice_headline(payload)) + elif kind in self._event_kinds: + log = await self._ensure_event_log() + log.write(f"[{kind}] {_render_event(payload)}") + + +class GenericRustApp(MultiJobApp[Any, GenericRustTaskHandler]): + """Textual TUI for a Rust application.""" + + def __init__( + self, + *, + phase_labels: dict[Any, str], + section_order: list[str], + header_text: str, + event_kinds: set[str], + notice_kinds: AbstractSet[str] = frozenset(), + ): + super().__init__( + phase_labels=phase_labels, section_order=section_order, header_text=header_text + ) + self._event_kinds = event_kinds + self._notice_kinds = notice_kinds + + def create_task_handler( + self, panel: VerticalScroll, info: TaskInfo[Any] + ) -> GenericRustTaskHandler: + return GenericRustTaskHandler( + info.task_id, info.label, panel, self, ToolDisplayConfig(), + self._event_kinds, self._notice_kinds, + ) + + def create_event_handler( + self, handler: GenericRustTaskHandler, info: TaskInfo[Any] + ) -> EventHandler: + return handler + + +class GenericRustConsoleHandler(MultiJobConsoleHandler[Any]): + """Stdout ``HandlerFactory`` for a Rust application.""" + + def __init__(self, event_kinds: set[str]): + super().__init__() + self._event_kinds = event_kinds + + @override + async def handle_event(self, payload: dict, path: list[str], checkpoint_id: str) -> None: + kind = payload.get("type") + if kind in self._event_kinds: + self._output(f"[{self._label(path)}] {kind}: {_render_event(payload)}") + + @override + async def handle_progress_event(self, payload: dict) -> None: + # Rust applications stream everything through Command::Emit (the custom + # channel); there are no progress-channel events. + pass diff --git a/composer/rustapp/host.py b/composer/rustapp/host.py new file mode 100644 index 00000000..ed495707 --- /dev/null +++ b/composer/rustapp/host.py @@ -0,0 +1,273 @@ +"""Assemble a Rust wheel into a runnable AutoProver application. + +The declarative descriptor lets a single host synthesize what a hand-written +application spells out (phase enum, core-phase mapping, artifact store, labels, +section order) and hand the driver a ready :class:`PipelineBackend`. + +* :func:`run_rust_pipeline` — the pipeline wrapper (build backend + ``PipelineRun`` + + call the shared driver). This is the piece a generic entry point calls. +* :func:`build_application` — bundle everything a frontend / ``main()`` needs + (the synthesized phase enum, labels, section order, and a backend factory). + +Applications that need a non-default store or backend class (e.g. Crucible's crate +store) pass ``store_factory`` / ``backend_cls``; the **same** phase enum is shared +by the frontend and the pipeline. +""" + +import asyncio +import importlib +from dataclasses import dataclass, field +from typing import Any, Callable, cast + +from composer.io.multi_job import HandlerFactory +from composer.pipeline.core import ( + DEFAULT_MAX_CPU_TASKS, + CorePipelineResult, + PipelineRun, + run_pipeline, +) +from composer.pipeline.ecosystem import ECOSYSTEMS, Ecosystem +from composer.rustapp.adapter import RustBackend +from composer.rustapp.descriptor import AppDescriptor +from composer.rustapp.phases import PhaseModel, build_phase_model +from composer.rustapp.result import RustFormalResult +from composer.rustapp.store import RustArtifactStore +from composer.rustapp.wire import CALLOUTS, AppArgs, RustAppModule, expect_payload, expect_text +from composer.sandbox.command import DEFAULT_TIMEOUT_S +from composer.sandbox.config import SandboxConfig +from composer.spec.artifacts import ArtifactStore +from composer.spec.context import SourceCode, WorkflowContext +from composer.spec.service_host import ServiceHost +from composer.tools.rag_env import validate_rag_db + +#: Build an artifact store for a run from the source + descriptor. +StoreFactory = Callable[[SourceCode, AppDescriptor], ArtifactStore[Any, RustFormalResult]] + + +@dataclass +class BackendOptions: + """Mutable run options closed over by :meth:`RustApplication.make_backend`. + + The CLI can adjust these (e.g. the sandbox) after building the application but + before :func:`run_application`, keeping one phase enum. Backend-specific tuning knobs + (e.g. a fuzz budget) travel as descriptor-declared args in :attr:`declared_args`. + """ + + command_timeout_s: int = DEFAULT_TIMEOUT_S + sandbox: SandboxConfig | None = None + #: Parsed values of the descriptor's declared CLI args, threaded into the backend and + #: injected into every component's ``AuthorInput.context``. Set by the entry point. + declared_args: dict[str, Any] = field(default_factory=dict) + + +def load_module(module_name: str) -> RustAppModule: + """Import a Rust application's compiled module by name (e.g. ``"echoprover"``). + + The cast is the one honest dynamic boundary in the host — nothing about ``import_module`` can be + checked statically. So every callout the host will call is verified *here* instead: a module that + isn't an AutoProver wheel, or is one built against an older SDK, fails at load with the missing + names listed rather than with an ``AttributeError`` several phases into a run.""" + module = importlib.import_module(module_name) + missing = [name for name in CALLOUTS if not callable(getattr(module, name, None))] + if missing: + raise TypeError( + f"{module_name!r} is not a complete AutoProver application module: it exports no " + f"{', '.join(missing)}. Rebuild the wheel against the current autoprover-sdk " + "(`export_app!` exports every callout)." + ) + return cast(RustAppModule, module) + + +def load_descriptor(module: RustAppModule) -> AppDescriptor: + """Parse a module's ``descriptor()`` JSON into an :class:`AppDescriptor`.""" + return AppDescriptor.model_validate_json(expect_payload(module.descriptor())) + + +def resolve_ecosystem(descriptor: AppDescriptor) -> Ecosystem[Any, Any, Any]: + """Resolve the descriptor's declared ecosystem against the registry.""" + eco = ECOSYSTEMS.get(descriptor.ecosystem) + if eco is None: + raise ValueError( + f"application {descriptor.name!r} selects ecosystem {descriptor.ecosystem!r}, " + f"which is not registered. Available: {sorted(ECOSYSTEMS)}." + ) + return eco + + +def _default_store(source: SourceCode, descriptor: AppDescriptor) -> RustArtifactStore: + return RustArtifactStore( + source.project_root, + descriptor.artifact_layout, + deliverable_mode=descriptor.deliverable_mode, + program=str(source.contract_name), + ) + + +def build_backend( + module: RustAppModule, + descriptor: AppDescriptor, + source: SourceCode, + *, + phases: PhaseModel, + store_factory: StoreFactory | None = None, + backend_cls: type[RustBackend] = RustBackend, + options: BackendOptions | None = None, +) -> RustBackend: + """Construct a :class:`RustBackend` around an already-built :class:`PhaseModel`. + + The model is required, not defaulted: a backend that synthesized its own would tag tasks with + enum members no frontend's labels are keyed by. :meth:`RustApplication.make_backend` is the + usual caller; this is the headless path. + """ + opts = options or BackendOptions() + sf = store_factory or _default_store + return backend_cls( + module=module, + descriptor=descriptor, + phases=phases, + store=sf(source, descriptor), + ecosystem=resolve_ecosystem(descriptor), + command_timeout_s=opts.command_timeout_s, + sandbox=opts.sandbox, + declared_args=opts.declared_args, + ) + + +async def run_rust_pipeline( + module_name: str, + source_input: SourceCode, + ctx: WorkflowContext[None], + handler_factory: HandlerFactory, + env: ServiceHost, + *, + max_concurrent: int = 4, + max_cpu_tasks: int = DEFAULT_MAX_CPU_TASKS, + max_bug_rounds: int = 3, + interactive: bool = False, +) -> CorePipelineResult[RustFormalResult]: + """Build the backend from ``module_name`` and run the shared driver — the Rust + analogue of ``run_autoprove_pipeline`` / ``run_foundry_pipeline``. + + This builds the application (and so its phase model) per call. It is the right entry for + headless callers whose handler ignores phases; for a TUI/console frontend, build a + :class:`RustApplication` once and use :func:`run_application`, so the frontend's labels and the + backend's phases come from the one model.""" + app = build_application(module_name) + return await run_application( + app, + source_input, + ctx, + handler_factory, + env, + max_concurrent=max_concurrent, + max_cpu_tasks=max_cpu_tasks, + max_bug_rounds=max_bug_rounds, + interactive=interactive, + ) + + +async def run_application( + app: "RustApplication", + source_input: SourceCode, + ctx: WorkflowContext[None], + handler_factory: HandlerFactory, + env: ServiceHost, + *, + max_concurrent: int = 4, + max_cpu_tasks: int = DEFAULT_MAX_CPU_TASKS, + max_bug_rounds: int = 3, + interactive: bool = False, +) -> CorePipelineResult[RustFormalResult]: + """Run a pre-built :class:`RustApplication`. The backend is constructed from the + app's already-synthesized phase enum, so the ``TaskInfo`` phases the driver emits + are the *same* enum members the frontend's ``phase_labels`` are keyed by — the + identity the frontend's label lookup relies on.""" + backend = app.make_backend(source_input) + run = PipelineRun( + ctx=ctx, source=source_input, _handler_factory=handler_factory, + _agent_semaphore=asyncio.Semaphore(max_concurrent), + _cpu_semaphore=asyncio.Semaphore(max_cpu_tasks), env=env, + ) + return await run_pipeline( + backend, run, ecosystem=app.ecosystem, interactive=interactive, threat_model=None, max_bug_rounds=max_bug_rounds + ) + + +@dataclass +class RustApplication: + """Everything a frontend / ``main()`` needs, synthesized from the descriptor. + + ``phases`` is the single :class:`PhaseModel` this application runs on — the backend's phases and + the frontend's labels both come from it, which is what keeps their members identical. + + ``options`` is mutable so the CLI can apply parsed flags (timeouts, sandbox) + before :func:`run_application` without rebuilding the phase model. + """ + + descriptor: AppDescriptor + module: RustAppModule + ecosystem: Ecosystem[Any, Any, Any] + phases: PhaseModel + options: BackendOptions = field(default_factory=BackendOptions) + store_factory: StoreFactory = field(default=_default_store) + backend_cls: type[RustBackend] = RustBackend + + @property + def name(self) -> str: + return self.descriptor.name + + @property + def header_text(self) -> str: + return self.descriptor.header_text + + def validate_preconditions(self, args: AppArgs) -> str | None: + """Delegate to the Rust precondition hook; return an error string or None.""" + return expect_text(self.module.validate_preconditions(args.model_dump_json())) + + def make_backend(self, source: SourceCode) -> RustBackend: + """Build the backend for this run — on this application's own :attr:`phases`.""" + return build_backend( + self.module, + self.descriptor, + source, + phases=self.phases, + store_factory=self.store_factory, + backend_cls=self.backend_cls, + options=self.options, + ) + + +def build_application( + module_name: str, + *, + store_factory: StoreFactory | None = None, + backend_cls: type[RustBackend] = RustBackend, + command_timeout_s: int = DEFAULT_TIMEOUT_S, + sandbox: SandboxConfig | None = None, +) -> RustApplication: + """Load a Rust wheel and synthesize a :class:`RustApplication`. + + ``store_factory`` / ``backend_cls`` let an application supply a specialized + store or prepared-system path (Crucible) while keeping one phase enum for the + frontend and the pipeline. + """ + module = load_module(module_name) + descriptor = load_descriptor(module) + ecosystem = resolve_ecosystem(descriptor) + # Both of the descriptor's registry references are resolved up-front, before the run spends + # anything: an unknown ecosystem or an unregistered RAG corpus is a wheel bug, not something to + # discover mid-run (an unavailable corpus, in contrast, degrades — see ``rag_env``). + validate_rag_db(descriptor.rag_db_default) + + return RustApplication( + descriptor=descriptor, + module=module, + ecosystem=ecosystem, + phases=build_phase_model(descriptor), + options=BackendOptions( + command_timeout_s=command_timeout_s, + sandbox=sandbox, + ), + store_factory=store_factory or _default_store, + backend_cls=backend_cls, + ) diff --git a/composer/rustapp/phases.py b/composer/rustapp/phases.py new file mode 100644 index 00000000..fc016dc4 --- /dev/null +++ b/composer/rustapp/phases.py @@ -0,0 +1,78 @@ +"""A descriptor's phase declarations, resolved into the one model an application runs on.""" + +import enum +from dataclasses import dataclass +from typing import Any, cast + +from composer.pipeline.core import CorePhases +from composer.rustapp.descriptor import AppDescriptor, PhaseRole, PhaseSpec + + +@dataclass(frozen=True) +class PhaseModel: + """The descriptor's phase declarations, resolved once into everything the pipeline and the + frontend need: the synthesized enum, the driver's core-phase mapping, the frontend's labels + and section order. + + :func:`build_phase_model` is the *only* place that synthesizes the enum, and every consumer + takes a built model rather than a descriptor. That is not tidiness: ``enum.Enum(...)`` mints a + fresh class per call, and both the frontend's label lookup and the driver's phase tagging match + members by identity — a second model built from the same descriptor would compare unequal to the + first and silently lose every label. + """ + + phase: type[enum.Enum] + #: The four slots the driver tags, as members of :attr:`phase`. + core: CorePhases + ordered: tuple[PhaseSpec, ...] + + @property + def labels(self) -> dict[Any, str]: + """Every declared phase's label, keyed by the enum member — what a frontend looks up.""" + return {self.phase[p.key]: p.label for p in self.ordered} + + @property + def section_order(self) -> list[str]: + """The labels in declared order — the frontend's section layout.""" + return [p.label for p in self.ordered] + + def member(self, key: str) -> enum.Enum: + """The member for a declared phase ``key``.""" + return self.phase[key] + + def role_member(self, role: PhaseRole) -> enum.Enum | None: + """The member of the phase claiming ``role``, or ``None`` when no phase claims it.""" + return next((self.phase[p.key] for p in self.ordered if p.role is role), None) + + @property + def first_member(self) -> enum.Enum: + """The first declared phase — where a task with no phase of its own is grouped.""" + return self.phase[self.ordered[0].key] + + +def build_phase_model(descriptor: AppDescriptor) -> PhaseModel: + """Synthesize an application's phase model from its descriptor. + + The enum is safe to synthesize: the code only ever uses phase members for ``.name`` and as dict + keys (no isinstance / identity checks against a static class). Every *required* role must be + claimed — the driver tags all four (the optional ones fall back; see :class:`PhaseRole`).""" + ordered = descriptor.ordered_phases() + name = "".join(part.capitalize() for part in descriptor.name.split("_")) + "Phase" + # enum.Enum's functional API is typed as returning an ``Enum`` instance, not the new + # class; it does return a class at runtime. + phase = cast(type[enum.Enum], enum.Enum(name, {p.key: p.key for p in ordered})) + + role_to_key = descriptor.role_map() + missing = [r.value for r in PhaseRole.required() if r not in role_to_key] + if missing: + raise ValueError( + f"descriptor {descriptor.name!r} is missing core phase(s): {missing}. " + "Every application must map analysis/extraction/formalization/report." + ) + core = CorePhases( + analysis=phase[role_to_key[PhaseRole.ANALYSIS]], + extraction=phase[role_to_key[PhaseRole.EXTRACTION]], + formalization=phase[role_to_key[PhaseRole.FORMALIZATION]], + report=phase[role_to_key[PhaseRole.REPORT]], + ) + return PhaseModel(phase=phase, core=core, ordered=tuple(ordered)) diff --git a/composer/rustapp/result.py b/composer/rustapp/result.py new file mode 100644 index 00000000..0bca23b4 --- /dev/null +++ b/composer/rustapp/result.py @@ -0,0 +1,95 @@ +"""The Rust backend's result type (``FormT``) and artifact identifier. + +``RustFormalResult`` is a plain pydantic model — the design's rule is that the +result stays Python-native so the driver's type-keyed cache +(``cache_get(formalizer.formalized_type)`` / ``cache_put``) round-trips it +unchanged. It is assembled by the adapter's loop from what the wheel published +(the wheel has no result type of its own: it answers per target, and the host +accumulates), so the only wire types in here are the per-check +:class:`~composer.rustapp.wire.Verdict` and the :class:`~composer.rustapp.wire.Target` objects a +gate run covered. It satisfies both ``FormalResult`` +(``artifact_text`` / ``commentary`` / ``property_checks()``) and +``ReportableResult`` (``skipped`` / ``property_checks()`` / ``output_link``) +structurally. +""" + +from dataclasses import dataclass + +from pydantic import BaseModel, Field + +from composer.rustapp.wire import Target, Verdict +from composer.authoring.state import SkippedProperty +from composer.spec.types import CheckName, PropertyTitle + + +class RustSetupSpec(BaseModel): + """The compiled shared setup spec (Crucible's fixture), wrapped for the store. + + It is one string, but the typed cache round-trips pydantic models only — and it earns a cache + entry: authoring + compiling it is a full LLM loop (on a large program, the longest single step + of a run), it is authored once for the whole run, and every component builds on it.""" + + source: str + + +class RustFormalResult(BaseModel): + """A successful Rust formalization. ``checks`` holds the property→check-names + map as JSON-friendly lists; ``property_checks()`` re-tuples it for the + protocols. The field is *not* named ``property_checks`` to avoid clashing with + that required method.""" + + commentary: str = "" + artifact_text: str = "" + checks: list[tuple[PropertyTitle, list[CheckName]]] = Field(default_factory=list) + skipped: list[SkippedProperty] = Field(default_factory=list) + output_link: str | None = None + # Per-check verdicts baked in at formalize time by a self-contained backend (check name -> the + # wheel's :class:`~composer.rustapp.wire.Verdict`, validated at the seam). Empty for + # run-service-backed backends (they use fetch_verdicts). + verdicts: dict[CheckName, Verdict] = Field(default_factory=dict) + # What the stamping gate run covered: each validation *target* — one invocation of the checker — + # with the checks it covered, in the order they ran. Several checks may share one target + # (Crucible puts a component's whole property set in one fuzz target), so this is neither + # ``checks``' keys nor its values. + # + # The names are mirrored into ``finalize``'s outcome set because a callout-mode wheel assembling + # one deliverable needs the real target names: re-deriving them from a display name would put + # the slug rule in two languages and smuggle a semantic value through a string. The checks are + # here so this component's coverage — which checks, and so which properties, these results are + # about — is answerable even where a whole target errored. + targets: list[Target] = Field(default_factory=list) + + def property_checks(self) -> list[tuple[PropertyTitle, list[CheckName]]]: + return [(title, list(names)) for title, names in self.checks] + + def check_properties(self) -> dict[CheckName, list[PropertyTitle]]: + """``checks`` inverted: check name -> the property titles it verifies. For display, where + the property's own words ("Balance never overflows") read better than the backend's check + name (``rule_balance_never_overflows``). + + A list, not one title: the mapping is many-to-many, and a check that discharges three + properties has no single title to be named after. A check absent here has none at all.""" + titles: dict[CheckName, list[PropertyTitle]] = {} + for title, names in self.checks: + for name in names: + titles.setdefault(name, []).append(title) + return titles + + +@dataclass(frozen=True) +class RustArtifact: + """Artifact identifier for a Rust backend — ``{prefix}_{slug}.{extension}``. + Prefix/extension come from the descriptor's ``ArtifactLayout`` so naming lives + in one place.""" + + slug: str + prefix: str + extension: str + + @property + def stem(self) -> str: + return f"{self.prefix}_{self.slug}" + + @property + def artifact_file(self) -> str: + return f"{self.stem}.{self.extension}" diff --git a/composer/rustapp/results.py b/composer/rustapp/results.py new file mode 100644 index 00000000..19f238bc --- /dev/null +++ b/composer/rustapp/results.py @@ -0,0 +1,118 @@ +"""Human-facing rollup of a Rust backend's per-check verdicts for the console / TUI. + +The canonical results artifact is ``report.json`` (the shared report phase). But — as with the +CVL and Foundry backends — the console/TUI otherwise surface only a counts block, so a completed +run reads as "success" with no visible verdicts. This turns the per-check verdicts baked into the +pipeline result (:attr:`RustFormalResult.verdicts`, published by ``validate``) into a compact +tally + per-check listing, using the report's own outcome labels so the wording matches the HTML +report. + +Backend-agnostic: the outcome wording is parametrized by the descriptor's ``backend_tag``, so any +Rust app whose results carry verdicts gets the same summary. +""" + +from collections import Counter +from dataclasses import dataclass +from typing import TYPE_CHECKING + +from composer.pipeline.core import CorePipelineResult, Delivered +from composer.rustapp.result import RustFormalResult +from composer.spec.source.report.render import outcome_glyph, outcome_label +from composer.spec.source.report.schema import Outcome, ReportBackend +from composer.spec.types import CheckName, PropertyTitle + +# ``RowName``: what a verdict row is called in the console/TUI listing. Phantom-typed like +# ``CheckName`` / ``PropertyTitle`` / ``ComponentName`` so it is a sibling of all three — the +# row is whatever :func:`_row_name` (or a component's display name) chose to show, and is never +# looked up as one of them. Defined here because the row is a display concept of this rollup, +# not an identity field of the analyzed system. +if TYPE_CHECKING: + class RowName(str): ... +else: + RowName = str + +# Tally display order — mirrors render.py's ``_OUTCOME_ORDER`` so the console and the HTML report +# list outcomes in the same sequence. +_ORDER = [Outcome.GOOD, Outcome.BAD, Outcome.TIMEOUT, Outcome.ERROR, Outcome.UNKNOWN] + + +@dataclass(frozen=True) +class CheckVerdict: + """One check's outcome: its display name and the neutral ``Outcome``.""" + + #: Whatever :func:`_row_name` chose to call the row — a property's title, a check's name, + #: or a component's display name. A :class:`RowName` so it belongs to none of those + #: namespaces and is never looked up as one of them. + name: RowName + outcome: Outcome + + +@dataclass(frozen=True) +class VerdictSummary: + """The delivered checks' verdicts, in pipeline order, plus the report backend tag for wording.""" + + verdicts: list[CheckVerdict] + backend_tag: ReportBackend + + @property + def counts(self) -> dict[Outcome, int]: + """Occurrence count per outcome, in display order, omitting absent outcomes.""" + c = Counter(v.outcome for v in self.verdicts) + return {o: c[o] for o in _ORDER if c.get(o)} + + @property + def tally(self) -> str: + """A one-line ``"10 No counterexample, 1 Counterexample"`` summary (backend labels).""" + return ", ".join( + f"{n} {outcome_label(self.backend_tag, o)}" for o, n in self.counts.items() + ) + + +def _row_name(check: CheckName, properties: list[PropertyTitle]) -> RowName: + """What to call one check's row: the property's own words when it verifies exactly one, and + otherwise the check's own name — the only thing that names the row unambiguously when one check + discharges several properties (or the author mapped none to it).""" + return RowName(properties[0] if len(properties) == 1 else check) + + +def summarize_verdicts( + result: CorePipelineResult[RustFormalResult], backend_tag: ReportBackend +) -> VerdictSummary: + """Extract the per-check verdicts baked into a completed run's ``outcomes``. + + One row per *check*, not per component: a component's gate run bakes a verdict per check it + covered, and all of them are rows (reading only the first would report one check where five + ran). Rows are named by :func:`_row_name`. + + Only *delivered* components carry verdicts; give-ups / exceptions are already surfaced in + ``result.failures`` and skipped here. A delivered component that bakes none at all (a + run-service-backed wheel, which reports through ``fetch_verdicts`` instead) still contributes + one UNKNOWN row, so the listing accounts for every delivered component.""" + verdicts: list[CheckVerdict] = [] + for o in result.outcomes: + if not isinstance(o.result, Delivered): + continue + formalized = o.result.result + if not formalized.verdicts: + verdicts.append(CheckVerdict(RowName(o.feat.display_name), Outcome.UNKNOWN)) + continue + titles = formalized.check_properties() + verdicts.extend( + CheckVerdict(_row_name(name, titles.get(name, [])), baked.outcome) + for name, baked in formalized.verdicts.items() + ) + return VerdictSummary(verdicts, backend_tag) + + +def format_verdict_lines(summary: VerdictSummary, *, indent: str = " ") -> list[str]: + """The ``Verdicts:`` tally line plus a per-check listing, in the console counts-block style. + Empty when nothing was delivered (the counts/failures block already conveys that).""" + if not summary.verdicts: + return [] + lines = [f"{indent}Verdicts: {summary.tally}"] + for v in summary.verdicts: + lines.append( + f"{indent} {outcome_glyph(v.outcome)} {v.name} — " + f"{outcome_label(summary.backend_tag, v.outcome)}" + ) + return lines diff --git a/composer/rustapp/session.py b/composer/rustapp/session.py new file mode 100644 index 00000000..44418bda --- /dev/null +++ b/composer/rustapp/session.py @@ -0,0 +1,1010 @@ +"""The Rust backend's authoring session — the shared workflow of :mod:`composer.authoring`, with a +wheel's callouts as its gate and its prompts. + +Two session kinds, differing only in what gates them and what they publish: + +* a **component** session authors one component's spec, gated by ``validate_spec`` (the wheel's + ``validate``, run per target), and publishes a property→checks mapping plus the verdicts the + gating run produced; +* a **setup** session authors the one shared spec every component builds on, gated by + ``compile_spec`` (the wheel's ``compile``). It formalizes no properties of its own, so it declares + no mapping and has nothing to skip. + +The wheel is still a passive service: it supplies prompts and answers the two blocking callouts. +What changed from a Python-driven retry loop is who holds the state — the agent does, in a buffer it +edits, and the gate is a tool it calls rather than a loop wrapped around it. +""" + +import asyncio +import json +import logging +from dataclasses import dataclass, field +from pathlib import Path +from typing import Annotated, Any, Callable, Literal, NotRequired, Sequence, override + +from langchain_core.tools import BaseTool +from langgraph.graph import MessagesState +from langgraph.types import Command +from pydantic import BaseModel, Field, create_model + +from graphcore.graph import FlowInput, tool_state_update +from graphcore.tools.schemas import ( + ToolFamilyParams, WithAsyncDependencies, WithAsyncImplementation, + WithInjectedId, WithInjectedState, tool_family, family_param +) + +from composer.authoring.buffer import ( + apply_spec_update, edit_spec_tool, get_spec_tool, +) +from composer.authoring.judge import ( + FeedbackThunk, JudgeBuilder, JudgeState, PropertyFeedbackProtocol, RebuttalBase, + build_feedback_judge, +) +from composer.authoring.state import ( + AuthoringExtra, MappingVocab, SkippedProperty, check_completion, make_validation_stamper, + merge_expected_failures, validate_check_mapping, +) +from composer.authoring.tools import give_up_tool, skip_tools +from composer.pipeline.core import GaveUp, PipelineRun +from composer.rustapp.descriptor import AppDescriptor +from composer.rustapp.result import RustFormalResult, RustSetupSpec +from composer.rustapp.wire import ( + AuthorInput, CompileOk, Prompt, RustAppModule, Target, Check, ValidateBuildFailed, + expect_payload, expect_text, parse_compile, parse_judge, parse_prompt, parse_validate, +) +from composer.rustapp.wire import Verdict as WireVerdict +from composer.sandbox.config import BackendSpec +from composer.spec.context import CacheKey, WorkflowContext +from composer.spec.types import CheckName, PropertyTitle +from composer.spec.gen_types import TypedTemplate +from composer.spec.graph_builder import run_to_completion +from composer.spec.service_host import ServiceHost +from composer.templates.loader import load_jinja_template +from composer.spec.source.report.schema import Outcome +from composer.ui.tool_display import ( + ToolDisplay, suppress_ack, tool_display, tool_display_of, tool_family_display, +) +from typing_extensions import TypedDict + +_log = logging.getLogger(__name__) + +VALIDATE_KEY = "validate" +COMPILE_KEY = "compile" +FEEDBACK_KEY = "feedback" + +_SKIP_DESCRIPTION = """ + Declare that you are skipping a property from the batch. + + You must provide the property's title and a justification. Skipping + excludes the property from the publish-time mapping check; only use + after a genuine attempt to formalize. + """ + +_SKIP_REASON = "Justification for why this property cannot be formalized" + + +@dataclass(frozen=True) +class CheckVocab: + """What this wheel calls one check when it talks to the model. + + Declared by the wheel (``AppDescriptor.check_noun``) because an author writes better when the + prompt speaks its own domain's language — Crucible's are harness functions, another backend's + are invariants. Only prose moves: the tools keep their ``check``-worded *names* so the protocol + can name them literally, and the wire keeps calling a check a check. Schema text is instantiated + via :class:`CheckNouns`; this object supplies the values and the leftover runtime strings.""" + + one: str + many: str + + @classmethod + def of(cls, descriptor: AppDescriptor) -> "CheckVocab": + return cls(one=descriptor.check_label(), many=descriptor.check_label(plural=True)) + + def fill(self, text: str) -> str: + """``text`` with ``{check}`` / ``{checks}`` rendered. For *runtime* strings (tool results, + the initial prompt). LLM-facing schemas go through :func:`tool_family` instead.""" + return text.format(check=self.one, checks=self.many) + + +class CheckNouns(ToolFamilyParams): + """The nouns :func:`tool_family` substitutes into the session tools' schemas.""" + + check: str + checks: str + + +class ProtocolParams(TypedDict): + """Render variables for the host-owned half of an author's system prompt.""" + gate_tool: str + has_judge: bool + has_checks: bool + check_noun: str + + +ProtocolTemplate = TypedTemplate[ProtocolParams]("authoring_protocol.j2") + + +# --------------------------------------------------------------------------- +# State +# --------------------------------------------------------------------------- + +@family_param(CheckNouns) +class PropertyCheckMapping(BaseModel): + """Maps one property from the batch to the {checks} that carry it. + + Many-to-many: a property may need several {checks}, and one {check} may carry several + properties (a single {check} discharging three related invariants).""" + property_title: PropertyTitle = Field( + description="The unique snake_case title of the property (from the batch listing) that " + "these {checks} verify" + ) + checks: list[CheckName] = Field( + description="The names of the {checks} in your spec that verify this property" + ) + + +class RustSpecExtra(AuthoringExtra): + #: What the author says verifies what, declared with ``map_checks`` and revisable. The distinct + #: check names in it are the set a gate run executes, and the whole of it is what the publish + #: gate validates. Empty for a setup session, which formalizes no properties of its own. + property_checks: list[PropertyCheckMapping] + expected_failures: Annotated[dict[CheckName, str], merge_expected_failures] + #: The verdicts the last full gating run produced, check name → the wheel's verdict. Recorded + #: verbatim: attribution is the wheel's, and the host does no verdict logic of its own. + verdicts: dict[CheckName, WireVerdict] + #: The targets that run covered, each carrying the checks it covered — the ground truth the + #: publish gate holds the mapping to, and what the result reports as this component's coverage. + ran: list[Target] + failed: bool | None + + +class RustSessionInput(RustSpecExtra, FlowInput): + pass + + +class RustSessionState(RustSpecExtra, MessagesState): + result: NotRequired[str] + + +_MAPPING = MappingVocab( + check_noun="check", + field_name="property_checks", + ran_source="the stamping validate_spec run", +) + + +def properties_of(mapping: Sequence[PropertyCheckMapping], check: CheckName) -> list[PropertyTitle]: + """The property titles the mapping says ``check`` verifies — several when one check discharges + several properties, none while the author has not said.""" + return [m.property_title for m in mapping if check in m.checks] + + +def declared_names(mapping: Sequence[PropertyCheckMapping]) -> list[CheckName]: + """The check names the author's mapping references, in first-seen order and without repeats. + + This is the set that runs. A name may be claimed by several properties (one check discharging + three invariants), so this is not the mapping flattened — it is its distinct names.""" + return list(dict.fromkeys(CheckName(n.strip()) for m in mapping for n in m.checks if n.strip())) + + +def declared_checks( + module: RustAppModule, input_json: str, mapping: Sequence[PropertyCheckMapping] +) -> list[Check]: + """:func:`declared_names` as :class:`Check`\\ s, each grouped by the wheel's ``target_for``. + + The parts come from the two parties that can know them: the *name* and what it verifies from the + author (the artifact is the author's, and so is the claim), the *grouping* from the wheel (which + invocation of the checker covers it, a backend convention).""" + return [ + Check( + name=name, + properties=properties_of(mapping, name), + target=expect_text(module.target_for(input_json, name)), + ) + for name in declared_names(mapping) + ] + + +# --------------------------------------------------------------------------- +# The gate tools +# --------------------------------------------------------------------------- + +@dataclass +class GateDeps: + """What both gate tools need to reach the wheel's blocking callouts.""" + + module: RustAppModule + input_json: str + workdir: Path + sandbox_json: str + emit: Callable[[str, dict], None] + #: What to call a check when talking to the author. + vocab: CheckVocab = CheckVocab("check", "checks") + #: Serializes the blocking callouts when the wheel shares one build dir across components. + command_sem: asyncio.Semaphore | None = None + + +async def _blocking(thunk: Callable[[], str], sem: asyncio.Semaphore | None) -> str: + """Run a wheel callout that spawns ``run-confined`` off the event loop, serialized by ``sem`` + when the wheel shares one workdir across concurrent components.""" + if sem is None: + return await asyncio.to_thread(thunk) + async with sem: + return await asyncio.to_thread(thunk) + + +def _first_line(s: str) -> str: + return next((ln for ln in s.splitlines() if ln.strip()), "").strip() + + +@tool_display("Building spec", "Build result") +class CompileSpec( + WithInjectedId, + WithInjectedState[RustSessionState], + WithAsyncDependencies[Command | str, GateDeps], +): + """ + Build your current spec with the real toolchain. + + A clean build stamps the publish gate for the buffer exactly as it now stands; any later + `put_spec` or `edit_spec` invalidates that stamp and you must build again. A failed build + returns the compiler's own diagnostics. + """ + + @override + async def run(self) -> Command | str: + spec = self.state["curr_spec"] + if spec is None: + return "No spec written yet — use `put_spec` first." + with self.tool_deps() as deps: + result = parse_compile( + await _blocking( + lambda: deps.module.compile( + deps.input_json, spec, str(deps.workdir), deps.sandbox_json + ), + deps.command_sem, + ) + ) + if not isinstance(result, CompileOk): + deps.emit("build_output", {"line": _first_line(result.errors) or "build failed"}) + return f"The build FAILED.\n\n{result.errors}" + return tool_state_update( + self.tool_call_id, + "The build succeeded.", + validations=make_validation_stamper(COMPILE_KEY)(self.state), + ) + + +@tool_family_display("Validating spec", "Validation result") +@tool_family(CheckNouns) +class ValidateSpec( + WithInjectedId, + WithInjectedState[RustSessionState], + WithAsyncDependencies[Command | str, GateDeps], +): + """ + Build your current spec and run the verifier over the {checks} you declared. + + Only the names `map_checks` listed are run — a {check} that is in the spec but not mapped is + not run — so declare before validating. Each {check} runs under its validation target; several + {checks} may share one, and each distinct target runs once. The report records the verdicts + verbatim — including a {check} the verifier could not find or never exercised, which does NOT + pass. + + The publish gate is stamped only by a run that covers EVERY declared {check} and comes back + clean — where a {check} you marked with `expect_check_failure` counts as clean. Naming + `checks` runs only those, which is for iterating on one problem; it never stamps. Any edit + after a stamping run invalidates the stamp. + """ + checks: list[CheckName] | None = Field( + default=None, + description="Run only the targets covering these {check} names. Omit to run everything, " + "which is what the publish gate requires.", + ) + + @override + async def run(self) -> Command | str: + spec = self.state["curr_spec"] + if spec is None: + return "No spec written yet — use `put_spec` first." + mapping = self.state["property_checks"] + with self.tool_deps() as deps: + vocab = deps.vocab + wanted = declared_checks(deps.module, deps.input_json, mapping) + if not wanted: + return ( + f"No {vocab.many} declared yet, so there is nothing to run. Use `map_checks` " + f"to say which {vocab.many} in your spec verify which property — only those " + f"names are run." + ) + if self.checks is not None: + asked = set(self.checks) + unknown = asked - {c.name for c in wanted} + if unknown: + return ( + f"Unknown {vocab.one} name(s): {', '.join(sorted(unknown))}. The declared " + f"{vocab.many} are: {', '.join(c.name for c in wanted)}." + ) + wanted = [c for c in wanted if c.name in asked] + covered = targets_of(wanted) + verdicts: dict[CheckName, WireVerdict] = {} + for target in covered: + res = parse_validate( + await _blocking( + lambda t=target: deps.module.validate( + deps.input_json, spec, t.model_dump_json(), + str(deps.workdir), deps.sandbox_json, + ), + deps.command_sem, + ) + ) + if isinstance(res, ValidateBuildFailed): + deps.emit( + "build_output", + {"line": _first_line(res.errors) or "build failed"}, + ) + return f"The build FAILED, so nothing was checked.\n\n{res.errors}" + for check, verdict in res.resolve(target): + verdicts[check.name] = verdict + _emit_verdict(deps, check, verdict) + + report = _verdict_report(verdicts, self.state["expected_failures"]) + partial = self.checks is not None + unexplained = _unexplained(verdicts, self.state["expected_failures"]) + if partial: + return f"{report}\n\nThis was a partial run, so it does not satisfy the publish gate." + if unexplained: + return ( + f"{report}\n\nThe publish gate is NOT satisfied: {', '.join(sorted(unexplained))} " + f"did not pass. Fix the spec, or — if the failure is the finding — mark the " + f"{vocab.one} with `expect_check_failure` and a reason." + ) + # ``ran`` travels with the stamp: the publish gate validates the mapping against the + # checks THIS run covered, and any later edit invalidates the stamp, so a stale set can + # never be the one publish is held to. + return tool_state_update( + self.tool_call_id, + f"{report}\n\nEvery declared {vocab.one} is accounted for; the publish gate is " + f"satisfied.", + verdicts=verdicts, + ran=covered, + validations=make_validation_stamper(VALIDATE_KEY)(self.state), + ) + + +def targets_of(checks: Sequence[Check]) -> list[Target]: + """``checks`` partitioned into the checker invocations that cover them — one :class:`Target` per + distinct target name, in first-seen order, each carrying its own checks. + + This is the whole of the run-vs-report split: several checks sharing a target means one build + and one run for all of them, while each still gets its own verdict. The host owns the grouping — + it decides what runs and in what order — so it hands the answer to the wheel rather than leaving + it to re-derive one.""" + names = list(dict.fromkeys(c.target_or_name() for c in checks)) + return [ + Target(name=name, checks=[c for c in checks if c.target_or_name() == name]) + for name in names + ] + + +def _emit_verdict(deps: GateDeps, check: Check, verdict: WireVerdict) -> None: + # The property's own words read better than a check name — the check carries the author's claim, + # so this needs nothing else. Several titles when one check discharges several properties. + name = ", ".join(check.properties) or check.name + line = f"{name}: {verdict.outcome.value}" + deps.emit( + "verdict", + { + "outcome": verdict.outcome.value, + "name": name, + "line": f"{line} — {verdict.detail}" if verdict.detail else line, + }, + ) + + +def _unexplained( + verdicts: dict[CheckName, WireVerdict], expected_failures: dict[CheckName, str] +) -> set[CheckName]: + """The checks that did not pass and were not marked as expected to fail — what stands between + the run and the publish gate.""" + return { + name for name, v in verdicts.items() + if v.outcome is not Outcome.GOOD and name not in expected_failures + } + + +def _verdict_report( + verdicts: dict[CheckName, WireVerdict], expected_failures: dict[CheckName, str] +) -> str: + lines = [] + for name, v in verdicts.items(): + mark = " (expected to fail)" if name in expected_failures else "" + detail = f" — {v.detail}" if v.detail else "" + lines.append(f" {name}: {v.outcome.value}{mark}{detail}") + return "Validation results:\n" + "\n".join(lines) + + +# --------------------------------------------------------------------------- +# Expected-failure marking +# --------------------------------------------------------------------------- + +def _expect_fail_label(p: dict, *, check: str, checks: str) -> str: + return f"Expecting {check} `{p['check_name']}` to fail" + + +@tool_family_display(_expect_fail_label, None) +@tool_family(CheckNouns) +class ExpectCheckFailure(WithAsyncImplementation[Command | str], WithInjectedId): + """ + Mark a {check} as expected to fail. + + Use this only when the failure IS the finding — the checker found a real counterexample, or the + property does not hold of the program under test. A marked {check} no longer blocks the publish + gate, and your reason is recorded with the result. Do not use it to get past your own bug. + """ + check_name: CheckName = Field(description="The name of the {check} expected to fail") + reason: str = Field( + description="Why this {check} is expected to fail — what the failure demonstrates" + ) + + @override + async def run(self) -> Command | str: + # An empty reason is the sentinel ``merge_expected_failures`` reads as an unmarking, so it + # must not get through here. + if not self.reason.strip(): + return "A non-empty reason is required when marking it as expected to fail." + return tool_state_update( + self.tool_call_id, "Recorded.", expected_failures={self.check_name: self.reason}, + ) + + +def _expect_pass_label(p: dict, *, check: str, checks: str) -> str: + return f"Expecting {check} `{p['check_name']}` to pass" + + +@tool_family_display(_expect_pass_label, None) +@tool_family(CheckNouns) +class ExpectCheckPassage(WithAsyncImplementation[Command], WithInjectedId): + """ + Unmark a {check} previously marked expected-to-fail. Every {check} is expected to pass by + default, so this only reverts a prior `expect_check_failure`. + """ + check_name: CheckName = Field(description="The name of the {check} now expected to pass after all") + + @override + async def run(self) -> Command: + return tool_state_update( + self.tool_call_id, "Recorded.", expected_failures={self.check_name: ""}, + ) + + +# --------------------------------------------------------------------------- +# Review +# --------------------------------------------------------------------------- + +def rebuttal_model(evidence_kinds: Sequence[str]) -> type[RebuttalBase]: + """The rebuttal type for a wheel, over the evidence it declared it can produce. + + Built per wheel rather than fixed because what counts as evidence is a property of the backend: + a fuzzing wheel can show a counterexample, a typechecking one only what its checker said.""" + kinds = tuple(evidence_kinds) or ("reasoned",) + return create_model( + "Rebuttal", + __base__=RebuttalBase, + __doc__=RebuttalBase.__doc__, + evidence_type=( + Literal[kinds], # type: ignore[valid-type] + Field( + description="What backs this rebuttal. Evidence from a tool outweighs an argument " + f"with the judge. One of: {', '.join(kinds)}." + ), + ), + ) + + +#: Appended to the wheel's judge system prompt. The wheel says what to review; the host says how the +#: verdict is returned, because the host is what reads it. +_JUDGE_PROTOCOL = ( + "\n\nRead the spec back with `get_spec` before judging it — reviewing the copy in this prompt " + "is not reviewing what was written. When you are done, call the `result` tool with `good` set " + "to whether the spec is acceptable as it stands, and `feedback` saying what to change (empty " + "when there is nothing to change). Your verdict is what the author's publish gate reads." +) + +class RustJudge: + """Phantom marker for the judge's child context.""" + + +def build_judge[K: (RustFormalResult, RustSetupSpec)]( + module: RustAppModule, + input_json: str, + *, + author_ctx: WorkflowContext[K], + env: ServiceHost, + backend_name: str, +) -> FeedbackThunk[RebuttalBase] | None: + """The wheel's judge as the shared feedback thunk, or ``None`` when it declares none for this + input — a wheel may review components and not the shared setup spec. + + Asked without a draft because it is asked before there is one: whether this input is reviewed + and who reviews it are both fixed for the session. What to ask about a particular draft is + ``judge_instruction``, below, once per round.""" + declared = module.judge(input_json) + if declared is None: + return None + system = ( + parse_judge(declared).system or "You are reviewing a formal specification." + ) + _JUDGE_PROTOCOL + + def apply_system(builder: JudgeBuilder) -> JudgeBuilder: + return builder.with_sys_prompt(system) + + def apply_prompt( + builder: JudgeBuilder, spec: str, + skipped: Sequence[SkippedProperty], rebuttals: Sequence[RebuttalBase], + ) -> JudgeBuilder: + return builder.with_initial_prompt(expect_payload(module.judge_instruction(input_json, spec))) + + def input_parts( + spec: str, skipped: Sequence[SkippedProperty], rebuttals: Sequence[RebuttalBase], + ) -> list[str | dict]: + parts: list[str | dict] = ["The proposed spec is", spec] + if skipped: + parts.append("The author declined to formalize these properties:") + for s in skipped: + parts.append(f" Property {s.property_title}: {s.reason}") + if rebuttals: + parts.append( + "The author has filed the following rebuttals against feedback from prior " + "rounds. Evidence produced by a tool carries near-binding weight; a reasoned " + "rebuttal is a conversation, not a veto." + ) + for i, r in enumerate(rebuttals, 1): + parts.append( + f" Rebuttal {i} [{getattr(r, 'evidence_type', 'reasoned')}]\n" + f" Addressing: {r.prior_feedback_reference}\n" + f" Evidence: {r.evidence}" + ) + return parts + + return build_feedback_judge( + # The judge reviews under its own child context so its memory namespace is disjoint from + # the author's — a reviewer that reads the author's notes is not independent. + ctx=author_ctx.child(CacheKey[K, RustJudge]("judge")), + env=env, + apply_system=apply_system, + apply_prompt=apply_prompt, + input_parts=input_parts, + readback=_readback(JudgeState), + description=f"{backend_name} spec review", + thread_prefix=f"{backend_name}-judge", + ) + + +@dataclass +class FeedbackDeps: + thunk: FeedbackThunk[RebuttalBase] + + +class FeedbackTool( + WithInjectedId, + WithInjectedState[RustSessionState], + WithAsyncDependencies[Command | str, FeedbackDeps], +): + """ + Send your current spec to the reviewer. + + The reviewer evaluates whether the spec really checks the properties it claims to, and whether + any skip is justified. An acceptance is recorded against the buffer exactly as it now stands; a + later edit invalidates it. + + If a prior-round suggestion was tried and provably does not work, file it in `rebuttals` with + the concrete evidence. Do not file rebuttals for feedback you merely disagree with — address + those by revising the spec. + """ + rebuttals: list[RebuttalBase] = Field( + default_factory=list, + description="Rebuttals to specific prior-round feedback, each identifying the point being " + "rebutted, classifying the evidence, and supplying it. Empty is the expected default.", + ) + + @override + async def run(self) -> Command | str: + spec = self.state["curr_spec"] + if spec is None: + return "No spec written yet — there is nothing to review." + with self.tool_deps() as deps: + res: PropertyFeedbackProtocol = await deps.thunk( + spec, self.state["skipped"], self.rebuttals, self.tool_call_id, + ) + body = f"Accepted? {res.good}\nFeedback:\n{res.feedback}" + if not res.good: + return body + return tool_state_update( + self.tool_call_id, body, + validations=make_validation_stamper(FEEDBACK_KEY)(self.state), + ) + + +# --------------------------------------------------------------------------- +# Publish +# --------------------------------------------------------------------------- + +@dataclass +class PublishDeps: + titles: list[PropertyTitle] + + +@tool_family_display( + lambda p, *, check, checks: f"Declaring {checks}", None +) +@tool_family(CheckNouns) +class MapChecks( + WithInjectedId, + WithInjectedState[RustSessionState], + WithAsyncImplementation[Command | str], +): + """ + Declare which {checks} in your spec verify which property. + + `validate_spec` runs only the names you list here — a {check} that is in the spec but not + mapped is not run — and the publish gate is held to this mapping. Declare before validating, + and call this again to revise — each call replaces the whole mapping. + + A property may need several {checks}, and one {check} may carry several properties: name it + under each. Do not name a property you skipped. + """ + property_checks: list[PropertyCheckMapping] = Field( + description="For every property you did NOT skip, the {checks} in your spec that verify it." + ) + + @override + async def run(self) -> Command | str: + names = declared_names(self.property_checks) + if not names: + return "That declares no checks at all. Name at least one per property you did not skip." + return tool_state_update( + self.tool_call_id, + f"Recorded: {len(self.property_checks)} propert" + f"{'y' if len(self.property_checks) == 1 else 'ies'} mapped onto " + f"{', '.join(names)}. `validate_spec` now runs only these names.", + property_checks=self.property_checks, + ) + + +@tool_display("Publishing spec", None) +class PublishSpec( + WithInjectedId, + WithInjectedState[RustSessionState], + WithAsyncDependencies[Command | str, PublishDeps], +): + """ + Publish your spec. Refused unless every required gate has accepted the buffer as it now stands, + and the mapping you declared accounts for every property that was not skipped. + """ + commentary: str = Field(description="Human-readable commentary on the spec you authored") + + @override + async def run(self) -> Command | str: + if (err := check_completion(self.state)) is not None: + return err + mapping = self.state["property_checks"] + with self.tool_deps() as deps: + # Ground truth is what the STAMPING run covered, not what is declared now: a name added + # since is one that did not run, and one removed is one that ran unclaimed. Both are + # errors here, which is why a mapping edit needs no stamp of its own. + err = validate_check_mapping( + [(m.property_title, m.checks) for m in mapping], + self.state["skipped"], + deps.titles, + _MAPPING, + ran=[c.name for t in self.state["ran"] for c in t.checks], + ) + if err is not None: + return err + return tool_state_update( + self.tool_call_id, "Accepted", + result=self.commentary, + failed=False, + ) + + +@tool_display("Publishing spec", None) +class PublishSetup( + WithInjectedId, + WithInjectedState[RustSessionState], + WithAsyncImplementation[Command | str], +): + """ + Publish the shared spec. Refused unless every required gate has accepted the buffer as it + now stands. + """ + commentary: str = Field(description="Human-readable commentary on the artifact you authored") + + @override + async def run(self) -> Command | str: + if (err := check_completion(self.state)) is not None: + return err + return tool_state_update( + self.tool_call_id, "Accepted", result=self.commentary, failed=False, + ) + + +_GIVE_UP_DESCRIPTION = """ + Give up on authoring this spec. + + A last resort, once you have exhausted the other tools. The reason is recorded and reported — + it is a better outcome than publishing something that only looks checked. + """ + + +# --------------------------------------------------------------------------- +# The session +# --------------------------------------------------------------------------- + +_GET_SPEC_DESCRIPTION = """ + Read back the current contents of your spec buffer. + """ + +_EDIT_SPEC_DESCRIPTION = """ + Make a surgical edit to the current spec instead of re-emitting the whole file. + + Provide `old_string` — an exact span copied from the current spec — and `new_string` to replace + it with. `old_string` must occur exactly once; include enough surrounding context to make it + unique. Any edit invalidates the stamps a gate tool or the reviewer put on the previous draft. + """ + + +def _readback(ty: type) -> BaseTool: + return get_spec_tool( + ty, + name="get_spec", + description=_GET_SPEC_DESCRIPTION, + missing="No spec written yet", + display=ToolDisplay("Reading spec", None), + ) + + +#: The wheel's put-time check, as the buffer's validator. +type SyntaxCheck = Callable[[str], str | None] + + +@tool_display( + label=lambda p: f"Putting spec ({len(p.get('spec', ''))} chars)", + result=suppress_ack("Spec write result", ("Accepted",)), +) +class PutSpec(WithAsyncDependencies[Command | str, SyntaxCheck], WithInjectedId): + """ + Replace the entire spec buffer with the source you provide. + + Prefer `edit_spec` once a draft exists — it is dramatically cheaper than re-sending the whole + file. Any write invalidates the stamps a gate tool or the reviewer put on the previous draft. + """ + spec: str = Field(description="The complete source of the spec file") + + @override + async def run(self) -> Command | str: + with self.tool_deps() as check: + return apply_spec_update( + tool_call_id=self.tool_call_id, text=self.spec, validator=check, + ) + + +def _syntax_check(module: RustAppModule, input_json: str) -> SyntaxCheck: + def check(spec: str) -> str | None: + return expect_text(module.check_syntax(input_json, spec)) + return check + + +@dataclass(frozen=True) +class SessionResult: + """What an authoring session produced.""" + + commentary: str + spec: str + skipped: list[SkippedProperty] + property_checks: list[tuple[PropertyTitle, list[CheckName]]] + verdicts: dict[CheckName, WireVerdict] + #: What the stamping gate run covered — the targets, each with its checks. + ran: list[Target] + expected_failures: dict[CheckName, str] + + +async def run_session[K: (RustFormalResult, RustSetupSpec)]( + *, + module: RustAppModule, + input: AuthorInput, + kind: Literal["component", "setup"], + titles: list[PropertyTitle], + env: ServiceHost, + ctx: WorkflowContext[K], + run: PipelineRun, + workdir: Path, + sandbox_dict: BackendSpec, + descriptor: AppDescriptor, + emit: Callable[[str, dict], None], + command_sem: asyncio.Semaphore | None = None, + description: str, +) -> SessionResult | GaveUp: + """Run one authoring session to completion and return what it published, or :class:`GaveUp`. + + Everything the wheel gets to say about how the session speaks and what it may cite comes from + ``descriptor``, so those declarations are read in one place rather than threaded as loose + strings.""" + input_json = input.model_dump_json() + backend_name = descriptor.name + vocab = CheckVocab.of(descriptor) + gate_deps = GateDeps( + module=module, + input_json=input_json, + workdir=workdir, + sandbox_json=json.dumps(sandbox_dict), + emit=emit, + command_sem=command_sem, + vocab=vocab, + ) + component = kind == "component" + judge = build_judge(module, input_json, author_ctx=ctx, env=env, backend_name=backend_name) + + gate_tool = "validate_spec" if component else "compile_spec" + required = [VALIDATE_KEY if component else COMPILE_KEY] + if judge is not None: + required.append(FEEDBACK_KEY) + + tools: list[BaseTool] = [ + *env.all_tools, + PutSpec.bind(_syntax_check(module, input_json)).as_tool("put_spec"), + _readback(RustSessionState), + edit_spec_tool( + RustSessionState, + name="edit_spec", + description=_EDIT_SPEC_DESCRIPTION, + missing="No spec written yet — use `put_spec` first.", + display=ToolDisplay("Editing spec", suppress_ack("Spec edit result")), + validator=_syntax_check(module, input_json), + reset_read=None, + ), + give_up_tool( + name="give_up", description=_GIVE_UP_DESCRIPTION, label=f"{backend_name} authoring", + ), + ctx.get_memory_tool(), + ] + if component: + tools += [ + _map_tool(vocab), + _validate_tool(gate_deps, vocab), + *skip_tools( + lambda: titles, + skip_description=_SKIP_DESCRIPTION, + skip_reason=_SKIP_REASON, + ), + *_expect_tools(vocab), + _publish_tool(PublishDeps(titles=titles)), + ] + else: + tools += [ + CompileSpec.bind(gate_deps).as_tool("compile_spec"), + PublishSetup.as_tool("result"), + ] + if judge is not None: + rebuttal = rebuttal_model(descriptor.evidence_kinds) + tools.append(_feedback_tool(judge, rebuttal)) + + prompt = parse_prompt(module.author_prompt(input_json)) + # The host owns the protocol half of the system prompt and the wheel the domain half, so a + # wheel never restates the tool contract — and cannot drift from it. + protocol = ProtocolTemplate.bind({ + "gate_tool": gate_tool, + "has_judge": judge is not None, + "has_checks": component, + "check_noun": vocab.one, + }).render_to(load_jinja_template) + system = f"{protocol}\n\n{prompt.system}" if prompt.system else protocol + + graph = ( + env.builder_heavy() + .with_state(RustSessionState) + .with_input(RustSessionInput) + .with_output_key("result") + .with_tools(tools) + .with_sys_prompt(system) + .with_initial_prompt(_initial_prompt(prompt, component, vocab)) + .compile_async() + ) + + tid, mnem = await ctx.thread_and_mnemonic() + state = await run_to_completion( + graph, + RustSessionInput( + input=[], + curr_spec=None, + skipped=[], + validations={}, + required_validations=required, + property_checks=[], + expected_failures={}, + verdicts={}, + ran=[], + failed=None, + ), + thread_id=tid, + recursion_limit=ctx.recursion_limit, + description=f"{description} ({mnem})", + ) + + assert "result" in state + assert state["failed"] is not None + if state["failed"]: + return GaveUp(reason=state["result"]) + spec = state["curr_spec"] + assert spec is not None + return SessionResult( + commentary=state["result"], + spec=spec, + skipped=state["skipped"], + property_checks=[(m.property_title, m.checks) for m in state["property_checks"]], + verdicts=state["verdicts"], + ran=state["ran"], + expected_failures=state["expected_failures"], + ) + + +def _validate_tool(deps: GateDeps, vocab: CheckVocab) -> BaseTool: + return ( + ValidateSpec.with_template(check=vocab.one, checks=vocab.many) + .bind(deps) + .as_tool("validate_spec") + ) + + +def _expect_tools(vocab: CheckVocab) -> list[BaseTool]: + return [ + ExpectCheckFailure.with_template(check=vocab.one, checks=vocab.many) + .as_tool("expect_check_failure"), + ExpectCheckPassage.with_template(check=vocab.one, checks=vocab.many) + .as_tool("expect_check_passage"), + ] + + +def _map_tool(vocab: CheckVocab) -> BaseTool: + # Formatting is not transitive: MapChecks.with_template rewrites this class's own text, but + # the nested PropertyCheckMapping would still show ``{checks}``. Template the element first, + # then splice it in. Display is applied after the splice so ``as_tool`` closes over the + # spliced schema, not the unspliced templated base. + return MapChecks.with_template(check=vocab.one, checks=vocab.many).as_tool("map_checks") + + +def _publish_tool(deps: PublishDeps) -> BaseTool: + return PublishSpec.bind(deps).as_tool("result") + + +def _feedback_tool(judge: FeedbackThunk[RebuttalBase], rebuttal: type[RebuttalBase]) -> BaseTool: + """``feedback_tool``, with the rebuttal type the wheel's declared evidence kinds produced.""" + schema = create_model( + "FeedbackTool", + __base__=FeedbackTool, + __doc__=FeedbackTool.__doc__, + rebuttals=(list[rebuttal], Field( # type: ignore[valid-type] + default_factory=list, + description=FeedbackTool.model_fields["rebuttals"].description, + )), + ) + tool_display_of(ToolDisplay("Getting feedback", "Feedback"))(schema) + return schema.bind(FeedbackDeps(thunk=judge)).as_tool("feedback_tool") + + +def _initial_prompt(prompt: Prompt, component: bool, vocab: CheckVocab) -> str: + """The wheel's instruction, plus the obligation the gate will hold the author to. + + The host states it rather than trusting each wheel's free-form instruction to: the names are the + author's to choose, but *that* every property is covered and every declared name is really in + the spec is enforced identically for every backend, so it is worded once here.""" + if not component: + return prompt.instruction + return ( + f"{prompt.instruction}\n\nDeclare with `map_checks` which {vocab.many} verify which " + f"property before you validate. `validate_spec` runs only the names you list there — a " + f"{vocab.one} that is in the spec but not mapped is not run. Every property must be " + f"verified by at least one {vocab.one} or skipped with a reason, every {vocab.one} you " + f"declare must really be in your spec, and one {vocab.one} may carry several properties." + ) diff --git a/composer/rustapp/store.py b/composer/rustapp/store.py new file mode 100644 index 00000000..1b59184b --- /dev/null +++ b/composer/rustapp/store.py @@ -0,0 +1,69 @@ +"""Artifact store for a Rust application. + +A thin :class:`composer.spec.artifacts.ArtifactStore` subclass: the base already +writes everything identical across backends (``properties.json``, +``commentary.md``, the property→checks map, ``token_usage.json``) and materializes +the artifact bytes from ``result.artifact_text``. All this subclass supplies is +the on-disk layout, taken from the descriptor's :class:`ArtifactLayout`. +""" + +from pathlib import Path +from typing import override + +from composer.rustapp.descriptor import ArtifactLayout, Callout, DeliverableMode, PerComponent +from composer.rustapp.result import RustArtifact, RustFormalResult +from composer.spec.artifacts import ArtifactStore +from composer.spec.util import ensure_dir + + +class RustArtifactStore(ArtifactStore[RustArtifact, RustFormalResult]): + """Persist a Rust backend's deliverables under the descriptor's layout. + + ``deliverable_mode`` selects how the *source* deliverable is written: + ``per_component`` (the default) writes one ``{prefix}_{slug}.{ext}`` file per component from + its ``artifact_text``; ``callout`` writes no per-component source — the wheel's ``finalize`` + renders the whole deliverable (e.g. Crucible's one shared crate). Either way the shared + metadata (``commentary.md`` / the property→checks map) is written per component.""" + + def __init__( + self, + project_root: str, + layout: ArtifactLayout, + *, + deliverable_mode: DeliverableMode | None = None, + program: str = "", + ): + self._layout = layout + self._deliverable_mode: DeliverableMode = deliverable_mode or PerComponent() + self._program = program + super().__init__( + project_root, + layout.property_suffix, + deliverable_dir=layout.deliverable_dir, + internal_dir=layout.internal_dir, + report_dir=layout.report_dir, + ) + + @override + def _artifact_dir(self) -> Path: + return ensure_dir(Path(self._project_root) / self._layout.artifact_dir) + + @override + def write_artifact(self, i: RustArtifact, artifact: RustFormalResult) -> Path: + """In ``callout`` mode, write only the shared metadata — the source files come from the + wheel's ``finalize`` — and return the deliverable's representative file as the component's + ``Delivered`` path: the descriptor's declared primary file, or the deliverable dir when + the app declared it has none. Otherwise defer to the base one-file-per-component writer.""" + mode = self._deliverable_mode + if not isinstance(mode, Callout): + return super().write_artifact(i, artifact) + self._write_commentary(i.stem, artifact.commentary) + self._write_property_map( + i.stem, self._property_suffix, dict(artifact.property_checks()) + ) + deliverable_path = mode.deliverable_path + return Path( + deliverable_path.format(program=self._program) + if deliverable_path + else self._layout.deliverable_dir + ) diff --git a/composer/rustapp/toolchain.py b/composer/rustapp/toolchain.py new file mode 100644 index 00000000..72885da7 --- /dev/null +++ b/composer/rustapp/toolchain.py @@ -0,0 +1,138 @@ +"""The seam for **the analyzed project's build system** — the two things the generic host wants to +know about a target it does not itself understand: where its code lives as a unit of that build +system, and how to prepare/build its workspace. + +Both are knowledge about the *ecosystem under analysis*, not about implementing a backend in Rust. An +application is written in Rust; the project it analyzes need not be — a Cargo crate's package and lib +names, an Anchor IDL, a Move package's named addresses are all one ecosystem's vocabulary. So the +framework declares this seam and the application that needs it registers an implementation per chain. +Like the ecosystem registry and :mod:`composer.tools.rag_env`, it is a declarative tag → one concrete +implementation, not an application fork: a chain's entry is shared by every wheel targeting it +(Crucible's fuzz harness and a future CVLR backend read a Cargo manifest and build a Solana program +the same way). + +**What crosses the seam is chain-shaped and opaque to everything here.** Both methods speak +``dict[str, Any]`` (Rust: ``autoprover_sdk::chain::ChainData``), typed at each end and nowhere in +between: the implementation registered here and the wheels targeting that chain share those types +through the chain's own support crate, while the host only transports them. Which type is inside +follows from the wheel's declared ``ecosystem``, not from inspecting keys. +That is what makes a new ecosystem a registration rather than an edit to +:mod:`composer.rustapp.wire`. + +**No chain has an entry yet.** The two methods are therefore reached in deliberately different ways: + +* :func:`source_unit` **degrades**. An empty answer is already a documented state — it is what a + language with no such unit yields, and what an unreadable layout yields — and the wheel fills the + gaps from its own convention (``SolanaSourceUnit::resolved``). So "no toolchain" is + indistinguishable from "nothing to resolve", which is the honest answer. +* :func:`project_toolchain` **raises**. A plan that only places files never asks for it (see + ``WorkspacePrep.needs_toolchain``), so reaching it means the wheel asked for work nothing can + perform; silently skipping it would surface much later as a mystifying compile error in the first + authored draft. Same treatment an unknown ecosystem or an unregistered RAG corpus gets, for the same + reason. + +The wheel never supplies a command line either way — only *what* to prepare (file contents, plus a +request its chain's toolchain understands) — so the network posture stays Python-owned. +""" + +from typing import Any, Protocol + +from composer.pipeline.ecosystem import ChainTag +from composer.rustapp.wire import AuthorInput, WorkspacePrep +from composer.sandbox.config import SandboxConfig +from composer.spec.context import SourceFields + + +class ProjectToolchain(Protocol): + """Everything the host needs of a project whose build system it does not understand. + + One object per chain rather than two registries, because both questions are the same knowledge: + the thing that can read a Cargo manifest is the thing that can drive Cargo.""" + + def source_unit(self, source: SourceFields) -> dict[str, Any]: + """Where ``source``'s code lives as a unit of this chain's build system — for Cargo, the crate + whose manifest owns the main source file, and what that crate is called. + + Returns the chain-shaped facts the wheel receives as ``AuthorInput.source_unit``; empty means + "nothing resolved, apply your own convention". Never raises: a layout it cannot read is that + same empty answer, not a failure.""" + ... + + async def prepare( + self, + plan: WorkspacePrep, + input: AuthorInput, + *, + source: SourceFields, + sandbox: SandboxConfig | None, + timeout_s: int, + ) -> dict[str, Any]: + """Execute the toolchain half of a ``workspace_prep`` plan. + + Called only when :attr:`~composer.rustapp.wire.WorkspacePrep.needs_toolchain`, and only after + the plan's ``files`` are in place — the manifest a warm or build reads is usually one of them. + + * ``plan`` — the wheel's parsed plan. :attr:`~composer.rustapp.wire.WorkspacePrep.toolchain_request` + is the field addressed to this call, in the shape this chain defines. + * ``input`` — the ``AuthorInput`` the plan was produced from, so the implementation can read + its own declared args out of ``args`` (Crucible passes ``program_idl`` there) without the + generic host having to know which keys mean anything. + * ``source`` — what is being analyzed. ``project_root`` is the workdir the host just wrote the + plan's ``files`` under and the root every build runs in; every path the implementation writes + below it must go through :func:`composer.rustapp.adapter.confined_target`, as the host's own + writes do. The rest of the fields are there so an implementation can resolve its own project + facts (the crate owning ``relative_path``, say) rather than being handed a shape the + framework would have to understand. + * ``sandbox`` — the run's confinement, or ``None`` when unsandboxed. Fetches run *unconfined* + (a fetch executes no untrusted code); anything that compiles runs confined + offline + (``docs/command-sandbox.md`` §5). + + Returns what the prep established, chain-shaped, for the host to report back to the wheel as + ``AuthorInput.prep_facts`` — empty when the plan asked for nothing it had to establish. + + As a concrete example — Crucible's fuzz harness on Solana — the request is a ``SolanaPrep``, + asking for three things, in order: + + * *warm*: ``cargo fetch`` each of ``warm_dirs`` — the harness crate's dir **and** the + program's own crate dir — into the run's private ``CARGO_HOME``, so the confined + + offline builds that follow find every dependency already present instead of dying on + the first download. + * *build*: ``cargo-build-sbf`` the program's lib target (``build_program``), leaving + ``target/deploy/.so`` in the workspace for the harness to load into LiteSVM. + * *IDL*: place the program's IDL at ``idl_dest`` — the operator's ``--program-idl`` file + when supplied, else the one ``anchor idl build`` emits — normalized to carry the + program's address. + + The warm and the build are side effects on the workspace; only the IDL becomes a *fact*. + What this call returns for Crucible is ``SolanaPrepFacts`` — today one field, ``idl``: the + project-root-relative path where the IDL landed.""" + ... + + +#: Registered implementations, by chain. An entry is added together with the module it binds — a tag +#: whose implementation doesn't exist would pass every check here and then fail at the first plan that +#: needs it. Empty is a working state: see the module docstring for what each method does then. +PROJECT_TOOLCHAINS: dict[ChainTag, ProjectToolchain] = {} + + +def source_unit(chain: ChainTag, source: SourceFields) -> dict[str, Any]: + """Where ``source``'s code lives as a build-system unit, per ``chain``'s toolchain — empty when + the chain has none, which the wheel reads as "apply your own convention".""" + toolchain = PROJECT_TOOLCHAINS.get(chain) + return toolchain.source_unit(source) if toolchain is not None else {} + + +def project_toolchain(chain: ChainTag) -> ProjectToolchain: + """The implementation for ``chain``. Raises when there is none: the wheel asked for preparation + nothing here can perform, and every alternative to failing now (skip it, establish nothing) + defers the same failure to a place where it reads as the authoring agent's fault.""" + toolchain = PROJECT_TOOLCHAINS.get(chain) + if toolchain is None: + known = sorted(PROJECT_TOOLCHAINS) + raise ValueError( + f"the wheel's workspace_prep asks to prepare the {chain} project, but no project " + f"toolchain is registered for that chain " + f"({f'registered: {known}' if known else 'none is registered yet'}). Register one in " + "composer.rustapp.toolchain.PROJECT_TOOLCHAINS, or have the plan place files only." + ) + return toolchain diff --git a/composer/rustapp/wire.py b/composer/rustapp/wire.py new file mode 100644 index 00000000..83863530 --- /dev/null +++ b/composer/rustapp/wire.py @@ -0,0 +1,577 @@ +"""Python mirror of the Rust SDK's **runtime** ABI — the payloads that cross the FFI on every call. + +Peer of :mod:`composer.rustapp.descriptor`, which mirrors the *declarative* half (the +``AppDescriptor`` a wheel exports once). Together they are the whole seam: every string that goes +into or comes out of a wheel is one of these models, so a field renamed in +``rust/autoprover-sdk/src/lib.rs`` fails here — at the boundary, naming the field — instead of +silently reading as ``""`` three call frames later. + +Keep the field names, defaults and tags in lockstep with ``rust/autoprover-sdk/src/lib.rs``. + +The two results are **tagged unions** on the Rust side (``#[serde(tag = "status")]`` / +``#[serde(tag = "kind")]``), so they are discriminated unions here: a +:class:`ValidateBuildFailed` carries ``errors`` and no verdicts, a :class:`ValidateVerdicts` the +reverse, and ``isinstance`` is what tells them apart. Neither can be asked for a field the other +owns. + +Both halves of this seam ship together, so a payload missing a field is never version skew — it is a +mirror that drifted. Nothing here is tolerant of that: **the side that deserializes requires +everything**. Python deserializes the *inbound* payloads, so those models default nothing and reject +unknown fields — a field a wheel omits, or one it sends that the host doesn't declare, fails in +``model_validate`` naming the field, at the callout that returned it. + +Defaults on the *outbound* models are a different thing wearing the same clothes. Python only +serializes those, and ``model_dump_json`` writes every field whether or not it was set, so a default +there costs the wire nothing and buys a constructor: an empty ``source_unit`` is how the host says it +resolved nothing. Requiring those payloads in full is the Rust side's job, and it does it the same +way — no ``#[serde(default)]``, ``deny_unknown_fields``, and ``crate::required::present`` on the +``Option`` fields serde would otherwise fill in silently. + +Three payloads on this seam are **chain-shaped**: ``source_unit``, ``prep_facts`` and +``WorkspacePrep.toolchain_request`` are typed here as bare ``dict[str, Any]`` (Rust: +``chain::ChainData``) because their fields belong to the *analyzed project's* build system, which this +framework deliberately holds no schema for — see :mod:`composer.rustapp.toolchain`. They are the same +treatment ``model`` and ``unit`` already get, and for the same reason. +""" + +import json +from collections import Counter +from typing import TYPE_CHECKING, Annotated, Any, Callable, Literal, Protocol, Self + +from pydantic import BaseModel, ConfigDict, Field, TypeAdapter + +from composer.spec.source.report.schema import Outcome +from composer.spec.types import CheckName, ComponentName, PropertyTitle, PropertyType + +# ``TargetName``: what a backend selects when it invokes the checker — one :class:`Target`'s name +# (Crucible: the component's harness fn / Cargo feature). Phantom-typed like the vocabulary in +# :mod:`composer.spec.types`, and defined here because targets exist only on this seam. A sibling +# of ``CheckName``: a check that shares no target is *named after* its check name +# (:meth:`Check.target_or_name`), but the two namespaces are otherwise distinct. +if TYPE_CHECKING: + class TargetName(str): ... +else: + TargetName = str + + +class WireModel(BaseModel): + """Base for every payload on this seam, and for :mod:`composer.rustapp.descriptor`'s. + + Rejects unknown fields, the counterpart of the Rust side's ``#[serde(deny_unknown_fields)]``: + a key one half sends and the other doesn't declare is drift between two mirrors that ship + together, so it should stop the run naming the key rather than be dropped on the floor.""" + + model_config = ConfigDict(extra="forbid") + + +# --------------------------------------------------------------------------- +# Outbound — what the host sends into a callout. +# --------------------------------------------------------------------------- + +class Property(WireModel): + """One property to formalize, the unit it was inferred for, and the ``slug`` the host assigned + it — unique within the batch, and what a backend names this property's :class:`Check` after + (Crucible: ``c_``).""" + + #: The unit whose analysis produced this property (``FeatureUnit.display_name``, the report's + #: component name). A title identifies a property only within its own unit, so a setup spec — + #: which is sent every unit's properties at once — needs this to tell two same-titled ones apart + #: and to know which unit's surface each has to be checkable against. + component: ComponentName + title: PropertyTitle + #: The shared vocabulary (:data:`~composer.spec.types.PropertyType`), mirrored by the Rust + #: ``PropertyKind`` — a closed set on both sides rather than a free string. + sort: PropertyType + description: str + slug: str = "" + + +class AppArgs(WireModel): + """The run's resolved inputs, as the two argument-shaped callouts (``validate_preconditions``, + ``sandbox_grants``) receive them. Mirrors the Rust ``AppArgs``. + + Every part the host already knows is its own field: ``program`` and ``source_path`` are the two + halves of the entry point's ``path:Name`` argument, split *here* so no wheel re-splits it.""" + + #: The project root, absolute. + project_root: str + #: The analysis identifier — never the name of a build-system unit (that is ``source_unit``). + program: str + #: The main source file, project-root-relative. + source_path: str + #: The design doc, when one was named on the command line. + system_doc: str | None = None + #: Where the analyzed code lives as a unit of its own build system, chain-shaped — see + #: :attr:`_AuthorInputBase.source_unit`. These two callouts run before workspace prep, so there + #: are no prep facts to accompany it. + source_unit: dict[str, Any] = Field(default_factory=dict) + #: The wheel's own declared flags, keyed by argparse dest (``--fuzz-timeout`` → + #: ``fuzz_timeout``). Untyped: the *wheel* declares these, so the host has no schema for them. + declared: dict[str, Any] = Field(default_factory=dict) + + +class _AuthorInputBase(WireModel): + """What every authoring/gating callout is told, whatever is being authored.""" + + #: The *analysis* identifier of the program under test — a label and a namespace, never the name + #: of a build-system unit (that is :attr:`source_unit`, and the two are independent). + program: str + #: Where the analyzed code lives as a unit of *its own* build system, resolved once per run by + #: the chain's registered :class:`~composer.rustapp.toolchain.ProjectToolchain` and carried + #: unchanged from here on. Chain-shaped and opaque to the host: a Cargo crate is a directory, a + #: package, a lib target and an Anchor requirement; a Move package is not. Empty when nothing was + #: resolved, which is when the wheel applies its own convention. + source_unit: dict[str, Any] = Field(default_factory=dict) + #: The properties this artifact must make checkable. + props: list[Property] = Field(default_factory=list) + #: The compiled shared setup spec, for a wheel that declared a + #: :class:`~composer.rustapp.descriptor.SetupSpec`. + setup: str | None = None + #: What workspace prep established from the wheel's own + #: :attr:`WorkspacePrep.toolchain_request` — chain-shaped like :attr:`source_unit`, and produced + #: by the same toolchain. Empty when the plan asked for nothing beyond placing files. A fact here + #: means *the thing it describes is in place*, which is what a wheel reads to decide how it + #: sources the program's types. + prep_facts: dict[str, Any] = Field(default_factory=dict) + #: The run's values for the wheel's own declared flags, keyed by argparse dest. Untyped: the + #: wheel declares them, so the host has no schema for them. + args: dict[str, Any] = Field(default_factory=dict) + + def with_props(self, props: list[Property]) -> Self: + """This input with ``props`` replaced — the setup spec's base input plus the properties + it has to make checkable, which only exist after extraction.""" + return self.model_copy(update={"props": props}) + + def with_prep_facts(self, prep_facts: dict[str, Any]) -> Self: + """This input with what the workspace prep just established (the preflight gate re-renders + the prep's input, and must see the workspace the prep actually set up).""" + return self.model_copy(update={"prep_facts": prep_facts}) + + +class PreflightInput(_AuthorInputBase): + """Gate the prepared workspace before anything is authored: the wheel renders its own skeleton + and ``compile`` builds it. Runs before analysis finishes, so it carries no model and no unit.""" + + kind: Literal["preflight"] = "preflight" + + +class SetupInput(_AuthorInputBase): + """Author the one shared spec every unit builds on, from the analyzed model and *every* unit's + properties.""" + + kind: Literal["setup"] = "setup" + #: The analyzed system model. Opaque to the host seam — its shape is the ecosystem's. + model: dict[str, Any] = Field(default_factory=dict) + #: Every unit the run is about to formalize (``FeatureUnit.feature_json()``). This is the only + #: callout that sees the set whole — a component turn holds one, a preflight runs before any + #: exists — so a wheel whose setup gate builds scaffolding the whole set implies (a manifest's + #: feature list, a crate root's module declarations) builds the real thing there. Deliberately + #: **not** part of the setup cache identity (:func:`composer.rustapp.adapter._setup_identity`): + #: it decides scaffolding, not what gets authored. + units: list[dict[str, Any]] = Field(default_factory=list) + + def with_units(self, units: list[dict[str, Any]]) -> Self: + """This input with the run's unit set attached — known at the same moment :meth:`with_props` + is applied, and kept separate for the same reason it is excluded from the cache identity.""" + return self.model_copy(update={"units": units}) + + +class ComponentInput(_AuthorInputBase): + """Author (and gate) one unit's spec.""" + + kind: Literal["component"] = "component" + #: The unit being formalized (``FeatureUnit.feature_json()``). Opaque to the host seam. + unit: dict[str, Any] = Field(default_factory=dict) + + +#: The input to every authoring/gating callout — tagged on ``kind`` (Rust ``Authored``). A variant +#: rather than one struct with a tag beside it: each kind carries something the other two have +#: nothing to say about, and none of them can be asked for another's payload. +AuthorInput = Annotated[ + PreflightInput | SetupInput | ComponentInput, Field(discriminator="kind") +] + + +class SkippedProperty(WireModel): + """A property the author declined to formalize, with its justification. Mirrors the Rust + ``SkippedProperty`` and carries the same fields as the host's own + :class:`composer.authoring.state.SkippedProperty`, which is what it is built from.""" + + property_title: PropertyTitle + reason: str + + +class Delivered(WireModel): + """What a component that reached the deliverable produced. Mirrors the Rust ``Delivered``.""" + + status: Literal["delivered"] = "delivered" + artifact_text: str = "" + #: The validation targets this component's checks ran under, in the order they ran — what a + #: callout-mode wheel keys its deliverable sections and declared features on. + targets: list[TargetName] = Field(default_factory=list) + property_checks: list[tuple[PropertyTitle, list[CheckName]]] = Field(default_factory=list) + #: What the author declined to formalize, and why — disjoint from :attr:`property_checks`. + skipped: list[SkippedProperty] = Field(default_factory=list) + unit_file: str | None = None + run_link: str | None = None + + +class ComponentGaveUp(WireModel): + """Formalization gave up on this component; it contributes nothing to the deliverable.""" + + status: Literal["gave_up"] = "gave_up" + + +#: A component's outcome — tagged on ``status`` (Rust ``ComponentOutcome``). A variant rather than a +#: ``delivered`` flag beside always-present fields: there is nothing to read on one that gave up. +ComponentOutcome = Annotated[Delivered | ComponentGaveUp, Field(discriminator="status")] + + +class FinalizeComponent(WireModel): + """One component's line in the ``finalize`` payload.""" + + name: ComponentName + outcome: ComponentOutcome + + +class FinalizeInput(WireModel): + """The full outcome set handed to ``finalize`` (Rust ``FinalizeInput``): everything a wheel + needs to render the whole deliverable, including the same project facts the gated builds used — + what ships must be what was checked.""" + + program: str + #: As every authoring callout received it — see :attr:`_AuthorInputBase.source_unit`. + source_unit: dict[str, Any] = Field(default_factory=dict) + #: As every authoring callout received it — see :attr:`_AuthorInputBase.prep_facts`. + prep_facts: dict[str, Any] = Field(default_factory=dict) + components: list[FinalizeComponent] = Field(default_factory=list) + #: The compiled shared setup spec, when the wheel declared one. + setup: str | None = None + + +# --------------------------------------------------------------------------- +# Inbound — what a callout returns. +# --------------------------------------------------------------------------- + +class Prompt(WireModel): + """An authoring instruction for one LLM turn, plus an optional backend-defined system prompt + (``None`` → the host's neutral default).""" + + instruction: str + system: str | None + + +class Judge(WireModel): + """The reviewer a wheel declares for an input — what is fixed for the whole authoring session, + which is why ``judge`` is asked once and without a draft. What to ask about a *particular* draft + is the per-round ``judge_instruction``.""" + + #: The domain half of the reviewer's system prompt (``None`` → the host's neutral role). The + #: host appends the review protocol either way. + system: str | None + + +class CompileOk(WireModel): + """The spec (or, in a preflight, the wheel's own skeleton) built.""" + + status: Literal["ok"] + + +class CompileFailed(WireModel): + """It did not build. ``errors`` is the diagnostics the wheel extracted, and becomes the revise + context for the next authoring turn.""" + + status: Literal["failed"] + errors: str + + +#: ``compile``'s result — tagged on ``status`` (Rust ``CompileResult``). +CompileResult = Annotated[CompileOk | CompileFailed, Field(discriminator="status")] + + +class Check(WireModel): + """One check the *author* declared: the backend's name for a runnable verification — a CVL rule, + a foundry test, a tagged fuzz assertion. A check yields a :class:`Verdict` and becomes one row of + the report. + + :attr:`properties` is what the author declared this check verifies — the mapping's own claim, + carried verbatim rather than guessed by the wheel, and never empty (a check exists *because* + something claimed it). Usually one title; several when one rule discharges several related + invariants. It is here because some checkers speak in properties rather than in check names + (Crucible tags each assertion message with its property title, so this is what lets it place a + counterexample), while a backend whose checker reports per check can ignore it. + + ``target`` names the :class:`Target` this check runs under — one invocation of the checker, + answered by the wheel's ``target_for``. Several checks may share one, so the host runs each + distinct target once and the wheel attributes the outcome back to each check.""" + + name: CheckName + properties: list[PropertyTitle] + target: TargetName | None + + # Mirrors the Rust ``Check::target_or_name``. + def target_or_name(self) -> TargetName: + """The target this check runs under — its own name unless it shares one.""" + return self.target or TargetName(self.name) + + +class Target(WireModel): + """One invocation of the checker — one build + one run — and the checks it covers, which is what + ``validate`` must return a verdict for. Mirrors the Rust ``Target``. + + Targets group *running*; checks group *reporting*. A target sits inside one unit's session, so + the three nest: a unit's checks partition into its targets. A backend that checks a whole + property set in one run therefore pays for one build and still reports a row per property. + + The host owns the grouping (it decides what to run, and in what order), so it hands the answer + over rather than leaving the wheel to recover it by re-deriving its own ``checks`` and filtering + them by name.""" + + #: What the backend selects when it runs the checker (Crucible: the component's harness fn, which is also its Cargo + #: feature). + name: TargetName + #: Usually one; several when a backend checks a whole property set in one run. + checks: list[Check] = Field(default_factory=list) + + +class Verdict(WireModel): + """One check's outcome. Mirrors the Rust ``Verdict`` and maps onto the report's + :class:`composer.spec.source.report.collect.Verdict` (whose ``message`` is this ``detail``).""" + + outcome: Outcome + line: int | None + duration_seconds: float | None + unit_file: str | None + #: Human-readable explanation of a non-GOOD outcome — the counterexample / assertion message for + #: a BAD, the error text for an ERROR. + detail: str | None + + @classmethod + def with_outcome(cls, outcome: Outcome) -> "Verdict": + """A bare verdict: the outcome, no diagnostics. Mirrors the Rust ``Verdict::with_outcome``, + and exists for the same reason — every field being required is right for the wire and no + reason for a caller that has only an outcome to spell four nulls to say so.""" + return cls(outcome=outcome, line=None, duration_seconds=None, unit_file=None, detail=None) + + +class ValidateBuildFailed(WireModel): + """The shared build failed, so nothing was checked — no check got a verdict.""" + + kind: Literal["build_failed"] + errors: str + + +class ValidateCoverageError(RuntimeError): + """``validate`` answered with a verdict set that is not the target's check set. + + A wheel bug, not a spec the author can fix, so it raises where a build failure returns a revise + prompt.""" + + +class ValidateVerdicts(WireModel): + """It built, and every check the target covers got a verdict, ``(check_name, verdict)``.""" + + kind: Literal["verdicts"] + verdicts: list[tuple[CheckName, Verdict]] + + def resolve(self, target: Target) -> list[tuple[Check, Verdict]]: + """Each of the target's checks paired with the verdict the wheel returned for it, in the + target's own order. + + A verdict is keyed by check name on the wire so a wheel picks from the checks the host sent + rather than restating them (a restated :class:`Check` could contradict the property→check + map the host published). Resolving that key here is what makes the pairing a `Check` for + everything upstream, and is the only place the docstring's "every check got a verdict" is + established: a name no check has, a check left without a verdict, or the same check twice + raises. Silence would be worse than a wrong verdict — a check whose verdict never arrives is + one the publish gate has nothing to object to, so an empty answer would stamp a component + nothing checked.""" + covered = [c.name for c in target.checks] + answered = Counter(name for name, _ in self.verdicts) + by_name = {name: verdict for name, verdict in self.verdicts} + missing = [n for n in covered if n not in by_name] + unknown = sorted(set(by_name) - set(covered)) + repeated = sorted(n for n, count in answered.items() if count > 1) + if missing or unknown or repeated: + raise ValidateCoverageError( + f"target {target.name!r} covers {covered}, but validate answered for " + f"{list(answered)}" + + (f"; no verdict for {missing}" if missing else "") + + (f"; no such check {unknown}" if unknown else "") + + (f"; more than one verdict for {repeated}" if repeated else "") + ) + return [(c, by_name[c.name]) for c in target.checks] + + +#: ``validate``'s result — tagged on ``kind`` (Rust ``ValidateOutcome``). +ValidateOutcome = Annotated[ValidateBuildFailed | ValidateVerdicts, Field(discriminator="kind")] + + +class WorkspacePrep(WireModel): + """A pure plan for preparing the workspace: the wheel declares it, the host executes it (so the + network posture stays Python-owned and the wheel never supplies a command line). + + Two halves, split by who can execute them — writing files is the same in every ecosystem, while + preparing a *project* means driving a build system the host does not understand.""" + + #: Files to write under the workdir, path-confined. Contents only. + files: dict[str, str] + #: What the chain's :class:`~composer.rustapp.toolchain.ProjectToolchain` should do beyond + #: writing :attr:`files` — warm a dependency cache, build the program, derive a client from it. + #: Chain-shaped and opaque here (Solana: ``{warm_dirs, build_program, idl_dest}``): the host + #: forwards it and only asks whether it is empty, which is what keeps a new ecosystem a + #: registration rather than a field on this model. Whatever it establishes comes back as + #: :attr:`_AuthorInputBase.prep_facts`. + toolchain_request: dict[str, Any] + + @property + def needs_toolchain(self) -> bool: + """Whether anything beyond writing :attr:`files` is asked for. A plan that only places files + is complete once they are written — nothing to warm or build, so nothing that needs a + toolchain, a sandbox or the network.""" + return bool(self.toolchain_request) + + +class SandboxGrants(WireModel): + """Extra grants a wheel needs unioned into the host-authored policy. Pure data — the wheel + declares grants, Python decides the policy.""" + + extra_ro: list[str] + #: Extra env *names* to pass through confinement. + extra_env: list[str] + + +class CalloutError(WireModel): + """Why a callout produced no payload. Mirrors the Rust ``CalloutError``. + + Success is the payload type unchanged; this is the only extra inbound JSON shape. A host bug + (bad input JSON, a serialize failure) is this, not an empty plan, a skipped review, or a + failed build the author should revise.""" + + kind: Literal["error"] + message: str + + +class CalloutFailed(RuntimeError): + """A callout could not produce its payload — the Python face of :class:`CalloutError`.""" + + def __init__(self, message: str) -> None: + super().__init__(message) + self.message = message + + +# --------------------------------------------------------------------------- +# The FFI surface itself. +# --------------------------------------------------------------------------- + +class RustAppModule(Protocol): + """The compiled wheel's module-level surface — what ``autoprover_sdk::export_app!`` exports. + + Members are declared as callables rather than methods so :data:`CALLOUTS` can be derived from + the annotations: the import-time check and this contract then cannot drift. Everything is + synchronous and speaks JSON strings; ``compile`` and ``validate`` block (they spawn + ``run-confined`` and release the GIL), which is why the host runs them off the event loop.""" + + #: ``() -> AppDescriptor`` JSON. The declarative spine. + descriptor: Callable[[], str] + #: ``(args_json) -> error | None``. A precondition the wheel checks before the run starts. + validate_preconditions: Callable[[str], str | None] + #: ``(input_json, check) -> target | None``. Which invocation a declared check runs under; + #: ``None`` makes it its own. Pure — the check *names* are the author's, and only the grouping + #: is the wheel's to say. + target_for: Callable[[str, str], TargetName | None] + #: ``(input_json) -> Prompt`` JSON. Asked once per authoring session — the session keeps its + #: own history, so there is no per-attempt revise prompt. + author_prompt: Callable[[str], str] + #: ``(input_json, spec) -> error | None``. Pure and cheap: the put-time gate on the buffer. + check_syntax: Callable[[str, str], str | None] + #: ``(input_json) -> Judge | None`` JSON. ``None`` ⇒ this wheel does not review this input. + #: Takes no spec: it is asked once, when the session is built, and neither answer depends on a + #: draft. + judge: Callable[[str], str | None] + #: ``(input_json, spec) -> str``. The instruction for one review round — the text itself, not + #: JSON. Called only for an input ``judge`` claimed. + judge_instruction: Callable[[str, str], str] + #: ``(input_json, spec | None, workdir, sandbox_json) -> CompileResult`` JSON. ``None`` is the + #: preflight, where nothing has been authored and the wheel builds its own skeleton — distinct + #: from an authored spec that happens to be empty. **Blocking.** + compile: Callable[[str, str | None, str, str], str] + #: ``(input_json, spec, target_json, workdir, sandbox_json) -> ValidateOutcome`` JSON, where + #: ``target_json`` is a :class:`Target` — the target to run and the checks it covers. + #: **Blocking.** + validate: Callable[[str, str, str, str, str], str] + #: ``(input_json) -> WorkspacePrep`` JSON. Pure — the host executes the plan. + workspace_prep: Callable[[str], str] + #: ``(args_json) -> SandboxGrants`` JSON. + sandbox_grants: Callable[[str], str] + #: ``(outcomes_json) -> {relpath: contents} | None`` JSON. + finalize: Callable[[str], str | None] + + +#: Every callout name the host may call, derived from :class:`RustAppModule` so the two can't drift. +#: Used to reject a module that isn't an AutoProver wheel (or is one built against an older SDK) +#: at load, with the missing callouts named. +CALLOUTS: tuple[str, ...] = tuple(RustAppModule.__annotations__) + + +# --------------------------------------------------------------------------- +# Parsing — one function per callout return, so every ``json.loads`` of a wheel's answer happens +# here and nowhere else. A malformed payload raises ``pydantic.ValidationError`` naming the field. +# A :class:`CalloutError` envelope raises :class:`CalloutFailed` before the payload parser runs. +# --------------------------------------------------------------------------- + +_COMPILE_RESULT: TypeAdapter[CompileOk | CompileFailed] = TypeAdapter(CompileResult) +_VALIDATE_OUTCOME: TypeAdapter[ValidateBuildFailed | ValidateVerdicts] = TypeAdapter(ValidateOutcome) +_FILES: TypeAdapter[dict[str, str]] = TypeAdapter(dict[str, str]) + + +def expect_payload(raw: str) -> str: + """Return ``raw`` unless it is the :class:`CalloutError` envelope, in which case raise + :class:`CalloutFailed`. + + Non-JSON text (a target name, a ``judge_instruction``, a precondition complaint) is returned + as-is: those channels are not JSON, and only an envelope is a wire-level failure.""" + try: + data = json.loads(raw) + except json.JSONDecodeError: + return raw + if not isinstance(data, dict) or data.get("kind") != "error": + return raw + raise CalloutFailed(CalloutError.model_validate(data).message) + + +def expect_text[T: str](raw: T | None) -> T | None: + """``None`` is a successful empty answer. A :class:`CalloutError` envelope raises.""" + if raw is None: + return None + expect_payload(raw) + return raw + + +def parse_compile(raw: str) -> CompileOk | CompileFailed: + return _COMPILE_RESULT.validate_json(expect_payload(raw)) + + +def parse_validate(raw: str) -> ValidateBuildFailed | ValidateVerdicts: + return _VALIDATE_OUTCOME.validate_json(expect_payload(raw)) + + +def parse_prompt(raw: str) -> Prompt: + return Prompt.model_validate_json(expect_payload(raw)) + + +def parse_judge(raw: str) -> Judge: + return Judge.model_validate_json(expect_payload(raw)) + + +def parse_workspace_prep(raw: str) -> WorkspacePrep: + return WorkspacePrep.model_validate_json(expect_payload(raw)) + + +def parse_sandbox_grants(raw: str) -> SandboxGrants: + return SandboxGrants.model_validate_json(expect_payload(raw)) + + +def parse_files(raw: str) -> dict[str, str]: + """``finalize``'s ``{relpath: contents}`` map.""" + return _FILES.validate_json(expect_payload(raw)) diff --git a/composer/sandbox/command.py b/composer/sandbox/command.py index 3c557272..b4bea275 100644 --- a/composer/sandbox/command.py +++ b/composer/sandbox/command.py @@ -1,4 +1,4 @@ -"""The local-command runner behind the ``RunCommand`` effect. +"""The local-command runner for trusted Python-side command execution. A single choke point: materialize a set of files into a workdir, run a command over them (as a child process, **never** a shell), and capture the result. The @@ -6,8 +6,8 @@ (A Rust backend's own ``compile``/``validate`` toolchain runs no longer go through this: they spawn the ``run-confined`` launcher directly from the wheel via -``autoprover_sdk::run_confined`` — see ``docs/rust-backend-api.md``. This runner and the -launcher share the same :mod:`composer.sandbox.policy` seam, which is why it lives in +``autoprover_sdk::sandbox::Workspace::run`` — see ``docs/rust-applications.md`` §8. This runner +and the launcher share the same :mod:`composer.sandbox.policy` seam, which is why it lives in :mod:`composer.sandbox` rather than under ``rustapp``.) Optional confinement is applied via a :class:`~composer.sandbox.policy.SandboxProvider` @@ -25,6 +25,7 @@ import contextlib import logging import os +import signal from dataclasses import dataclass from pathlib import Path, PurePosixPath from typing import TypedDict @@ -143,6 +144,7 @@ async def _run() -> CommandResult: env=child_env, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE, + start_new_session=True, ) except FileNotFoundError: return CommandResult(NOT_FOUND_EXIT, "", f"{spec.argv[0]}: not found on PATH") @@ -151,7 +153,12 @@ async def _run() -> CommandResult: proc.communicate(), timeout=timeout_s ) except asyncio.TimeoutError: - proc.kill() + # Process group (pgid == pid via start_new_session) so descendants + # die with the leader. killpg before wait: do not reap then reuse a pid. + try: + os.killpg(proc.pid, signal.SIGKILL) + except ProcessLookupError: + pass await proc.wait() return CommandResult(-1, "", f"command timed out after {timeout_s}s") rc = proc.returncode if proc.returncode is not None else -1 diff --git a/composer/sandbox/config.py b/composer/sandbox/config.py index b9cfb84d..20fd44a5 100644 --- a/composer/sandbox/config.py +++ b/composer/sandbox/config.py @@ -15,7 +15,11 @@ from dataclasses import dataclass from importlib.metadata import entry_points from pathlib import Path -from typing import NotRequired, Self, TypedDict, Unpack +from typing import Annotated, NotRequired, Self, TypedDict, Unpack + +# A three-file, zero-dependency package (and already a transitive dep via pydantic): +# the vocabulary for a field constraint, without pulling pydantic into this module. +from annotated_types import Ge from composer.sandbox.policy import SandboxPolicy, SandboxProvider, ensure_available from composer.sandbox.recipes import DEFAULT_ENV_PASSTHROUGH, rust_build_policy @@ -35,7 +39,10 @@ class BackendSpec(TypedDict): shape (``docs/command-sandbox.md`` §4).""" argv_prefix: list[str] - timeout_s: int + #: Wall-clock budget for the command. Bounded below because the mirrored field is a Rust `u64`: + #: a negative here is refused by the wheel's deserializer, so say so on this side rather than + #: leaving the two halves to disagree about the domain. + timeout_s: Annotated[int, Ge(0)] class SandboxArgs(TypedDict): @@ -112,7 +119,7 @@ def build_policy(self, workdir: str | Path) -> SandboxPolicy | None: async def backend_spec(self, workdir: str | Path, *, timeout_s: int) -> BackendSpec: """The ``Sandbox`` JSON a Rust backend's ``compile``/``validate`` consume to launch - a confined command (`autoprover_sdk::Sandbox`). Python keeps ownership of the + a confined command (`autoprover_sdk::sandbox::Sandbox`). Python keeps ownership of the confinement *intent* (this policy) and of translating it into an argv wrapper; the backend only prepends ``argv_prefix`` to its command — it names no sandbox mechanism. diff --git a/composer/sandbox/launcher.py b/composer/sandbox/launcher.py index 0ea7ab62..35bcf509 100644 --- a/composer/sandbox/launcher.py +++ b/composer/sandbox/launcher.py @@ -16,6 +16,7 @@ import asyncio import os import shutil +import sysconfig from pathlib import Path from composer.sandbox.policy import ( @@ -30,18 +31,23 @@ def _resolve_binary() -> str | None: - """Locate the ``run-confined`` binary: ``$RUN_CONFINED_BIN`` → ``PATH`` → the - dev build under ``rust/target/release`` (repo-relative). ``None`` if unbuilt.""" + """Locate the ``run-confined`` binary: ``$RUN_CONFINED_BIN`` → ``PATH`` → this + interpreter's scripts dir. ``None`` if unbuilt. + + The env var is for deployments that mount the binary elsewhere (the sandbox compose + overlay). The two probes both target the normal development install: + ``rust/run-confined`` builds as a maturin *bin* wheel that ``uv sync`` puts in + ``.venv/bin`` (see the ``apps`` dependency group) — PATH finds it under `uv run` or + an activated venv, and ``sysconfig`` finds it when the venv's interpreter is invoked + by path without activation.""" override = os.environ.get("RUN_CONFINED_BIN") if override and Path(override).is_file(): return override on_path = shutil.which(_BIN_NAME) if on_path: return on_path - # Dev fallback: composer/sandbox/launcher.py → repo root is parents[2]. - repo_root = Path(__file__).resolve().parents[2] - cand = repo_root / "rust" / "target" / "release" / _BIN_NAME - return str(cand) if cand.is_file() else None + in_venv = Path(sysconfig.get_path("scripts")) / _BIN_NAME + return str(in_venv) if in_venv.is_file() else None class LauncherProvider: @@ -61,8 +67,8 @@ def binary(self) -> str | None: async def available(self) -> Availability: if self._binary is None: return Reason( - f"{_BIN_NAME} binary not found; build rust/run-confined " - f"(cargo build -p run-confined --release) or set RUN_CONFINED_BIN" + f"{_BIN_NAME} binary not found; run `uv sync` (builds it from " + f"rust/run-confined into .venv/bin) or set RUN_CONFINED_BIN" ) try: proc = await asyncio.create_subprocess_exec( diff --git a/composer/sandbox/recipes.py b/composer/sandbox/recipes.py index da6ad12f..7bb5d227 100644 --- a/composer/sandbox/recipes.py +++ b/composer/sandbox/recipes.py @@ -112,8 +112,11 @@ def rust_build_policy( root — see :func:`shared_cargo_ro_paths`), Solana platform-tool directories, the system dirs, and ``extra_ro`` read-only. Non-existent paths are dropped. - With ``offline`` (the default — the sandbox has no network, §5), ``CARGO_NET_OFFLINE=1`` - is set in the child env. That one var forces *every* cargo invocation offline, + With ``offline`` (the default — the sandbox has no network, §5), + ``CARGO_NET_OFFLINE=true`` is set in the child env. Spelled ``true`` because cargo parses + this one as a config boolean and rejects anything else — ``=1`` aborts the build with + "provided string was not `true` or `false`" *while still going to the network*, so the + truthy-looking value is worse than no value at all. That one var forces every cargo offline, including the nested ``cargo`` that ``crucible run`` spawns to build the harness — so the deps must already be warm in the private ``CARGO_HOME`` (see :func:`warm_cargo_cache`, run *outside* the sandbox first). @@ -142,7 +145,7 @@ def rust_build_policy( env = {name: os.environ[name] for name in env_passthrough if name in os.environ} if offline: - env["CARGO_NET_OFFLINE"] = "1" + env["CARGO_NET_OFFLINE"] = "true" # A private temp dir UNDER the (writable) workdir, so tools that need scratch space # — notably the linker, which writes to $TMPDIR (default /tmp) during `cargo build` — # work without granting the shared /tmp (which may hold host/other-run secrets and diff --git a/composer/scripts/budget_math.py b/composer/scripts/budget_math.py new file mode 100644 index 00000000..26a6bf1c --- /dev/null +++ b/composer/scripts/budget_math.py @@ -0,0 +1,459 @@ +"""Budget calibration + live-test matrix generation from autoprove run trails. + +Reconstructs, per LLM call, exactly what ``CostAccumulator`` would have accrued — +``graphcore.utils.get_normalized_token_usage`` for the token buckets and +``composer.llm.pricing`` for the rates — at both cache-write TTLs (the trail doesn't +record which TTL a conversation used, so both bounds are shown; calibration uses the +1h bound, since the authors run with the long cache). Sub-agent threads +(``from_tool_id``) fold into their root thread, matching how budget scopes accrue: a +sub-agent spends from its parent's cost center. Phase attribution comes straight from +``ThreadMeta.cost_center`` (the thread logger stamps the ambient named-budget scope +into every thread record, budget or no budget); trails recorded before cost-center +tracking existed have no such field and classify entirely as unattributed. + +The run trail comes from one of: + + * an ``ap-trail export`` dump — ``.json.gz`` (gzipped) or plain ``.json``; + * a bare run id, fetched live from the store/checkpointer DB (the same wiring as + ``ap-trail export``), honoring ``--uid`` / ``$AUTOPROVER_USER_ID``. + +``--emit-matrix DIR`` additionally writes the calibrated live-test budget matrix — +ready-made ``--budget`` files plus a ``manifest.md`` stating each test's expected +outcome and the observables checklist: + + t1_control.json ample everything: must behave like an unbudgeted run + t2_formalization_curtail.json trips the component author mid-batch + t3_preparation_curtail.json trips the invariant author; run degrades gracefully + t4_pool_pressure.json trips the shared pool partway through the run + (T5, the caching interplay test, reuses t2 across two runs — see the manifest) + +Calls whose model has no pricing-table entry accrue ZERO cost in production, so a +budget can never trip on them — the script warns loudly when it sees any. + +Usage: + uv run scripts/budget_math.py [--uid U] + [--cap-fraction 0.75] [--wrapup-turns 4] [--emit-matrix DIR] +""" + +import argparse +import asyncio +import json +import sys +from dataclasses import dataclass +from pathlib import Path + +from langchain_core.messages import AIMessage + +from graphcore.utils import get_normalized_token_usage + +from composer.diagnostics.budget import BUDGET_PRESSURE_THRESHOLD +from composer.io.run_index import ExportedMessage, ExportedRun, read_export +from composer.io.thread_logging import ThreadMeta +from composer.llm.pricing import price_per_mtok +from composer.pipeline.ptypes import PhaseBudget + +PHASES: tuple[str, ...] = tuple(PhaseBudget.__annotations__) +UNATTRIBUTED = "unattributed" + + +def _phase_of(meta: ThreadMeta) -> str: + """The phase a thread accrued to, straight from its recorded ``cost_center``. + ``None`` (work outside any named scope: the pool, or pre-pipeline code) and any + center that isn't a `PhaseBudget` phase land in UNATTRIBUTED.""" + cc = meta.get("cost_center") + return cc if cc in PHASES else UNATTRIBUTED + + +# --------------------------------------------------------------------------- +# Per-call costing (mirrors CostAccumulator._cost_of at both cache-write TTLs) +# --------------------------------------------------------------------------- + +@dataclass(frozen=True) +class Call: + model: str | None + fresh_input: int + cache_read: int + cache_write: int + output: int + cost_5m: float + cost_1h: float + + @property + def priced(self) -> bool: + return self.cost_1h > 0 or not (self.fresh_input or self.cache_read or self.output) + + +def _cost_of(msg: AIMessage) -> Call | None: + usage = get_normalized_token_usage(msg) + total_input = usage["total_input_tokens"] + output = usage["total_output_tokens"] + if not (total_input or output): + return None + cache_read = usage["cache_read_tokens"] + cache_write = usage["cache_write_tokens"] + fresh = max(0, total_input - cache_read - cache_write) + model = usage["model_name"] + tier = price_per_mtok(model, total_input) + if tier is None: + return Call(model, fresh, cache_read, cache_write, output, 0.0, 0.0) + + def price(write_rate: float) -> float: + return ( + fresh * tier.input + + cache_read * tier.cache_read + + cache_write * write_rate + + output * tier.output + ) / 1_000_000 + + return Call(model, fresh, cache_read, cache_write, output, + price(tier.cache_write), price(tier.cache_write_1h)) + + +# --------------------------------------------------------------------------- +# Root folding + aggregation +# --------------------------------------------------------------------------- + +@dataclass +class Root: + description: str + phase: str + members: list[str] + calls: list[Call] + + def cost(self, long_cache: bool = True) -> float: + return sum(c.cost_1h if long_cache else c.cost_5m for c in self.calls) + + +def _fold_roots(run: ExportedRun) -> list[Root]: + """Fold sub-agent threads into their spawning root via ``from_tool_id`` chains.""" + owner_of_tool: dict[str, int] = {} + for i, t in enumerate(run.threads): + for entry in t.timeline: + if isinstance(entry, ExportedMessage) and isinstance(entry.data, AIMessage): + for tc in entry.data.tool_calls: + if t_id := tc.get("id"): + owner_of_tool[t_id] = i + + def root_of(idx: int) -> int: + seen: set[int] = set() + while idx not in seen: + seen.add(idx) + from_tool = run.threads[idx].meta.get("from_tool_id") + if from_tool is None or from_tool not in owner_of_tool: + return idx + idx = owner_of_tool[from_tool] + return idx + + groups: dict[int, list[int]] = {} + for i in range(len(run.threads)): + groups.setdefault(root_of(i), []).append(i) + + roots: list[Root] = [] + for root_idx, members in sorted(groups.items()): + meta = run.threads[root_idx].meta + calls: list[Call] = [] + for i in members: + for entry in run.threads[i].timeline: + if isinstance(entry, ExportedMessage) and isinstance(entry.data, AIMessage): + if (c := _cost_of(entry.data)) is not None: + calls.append(c) + roots.append(Root( + description=meta["description"], + phase=_phase_of(meta), + members=[run.threads[i].meta["description"] for i in members], + calls=calls, + )) + return roots + + +# --------------------------------------------------------------------------- +# Input loading +# --------------------------------------------------------------------------- + +async def _fetch_from_db(run_id: str, uid: str | None) -> ExportedRun: + # Imported lazily: pulls the full service stack, which file-based invocations skip. + from composer.io.run_index import build_export + from composer.workflow.services import checkpointer_context, store_context + async with store_context() as store, checkpointer_context() as checkpointer: + return await build_export(store, checkpointer, run_id, uid=uid) + + +def load_run(source: str, uid: str | None) -> ExportedRun: + p = Path(source) + if p.exists(): + if p.name.endswith(".gz"): + return read_export(str(p)) + return ExportedRun.model_validate_json(p.read_text()) + return asyncio.run(_fetch_from_db(source, uid)) + + +# --------------------------------------------------------------------------- +# Matrix generation +# --------------------------------------------------------------------------- + +def _cap(x: float) -> float: + """Round a computed cap to cents, never below one cent (a 0.00 cap means + 'curtail immediately', which is a test in itself but not what calibration wants).""" + return max(0.01, round(x, 2)) + + +@dataclass +class Matrix: + """The calibrated budget files, as {filename: {total, caps}} plus manifest prose.""" + files: dict[str, dict] + manifest: str + + +def build_matrix( + run: ExportedRun, + roots: list[Root], + *, + cap_fraction: float, + wrapup_turns: int, +) -> Matrix: + theta = BUDGET_PRESSURE_THRESHOLD + run_total = sum(r.cost() for r in roots) + phase_spend = {p: sum(r.cost() for r in roots if r.phase == p) for p in (*PHASES, UNATTRIBUTED)} + + # The author proxy: the priciest formalization root, falling back to the priciest + # root overall (e.g. a cache-warm source run where only invariant work ran live — + # same agent shape, stated as a proxy in the manifest). + formalization_roots = [r for r in roots if r.phase == "formalization"] + author = max(formalization_roots or roots, key=lambda r: r.cost()) + author_is_proxy = not formalization_roots + author_cost = author.cost() + per_call = [c.cost_1h for c in author.calls] + max_call = max(per_call, default=0.0) + late = per_call[-max(1, len(per_call) // 4):] + late_mean = sum(late) / len(late) if late else 0.0 + + ample = max(10.0, round(20 * run_total)) + ample_caps = {p: ample for p in PHASES} + + t2_cap = _cap(cap_fraction * author_cost) + prep_spend = phase_spend["formalization_preparation"] + t3_cap = _cap(theta * prep_spend) if prep_spend > 0 else None + t4_total = _cap(theta * run_total) + + files: dict[str, dict] = { + "t1_control.json": {"total": ample, "caps": dict(ample_caps)}, + "t2_formalization_curtail.json": { + "total": ample, "caps": {**ample_caps, "formalization": t2_cap}, + }, + } + if t3_cap is not None: + files["t3_preparation_curtail.json"] = { + "total": ample, "caps": {**ample_caps, "formalization_preparation": t3_cap}, + } + files["t4_pool_pressure.json"] = {"total": t4_total, "caps": dict(ample_caps)} + + def headroom_note(cap: float) -> str: + headroom = (1 - theta) * cap + if not max_call: + return "no per-call data" + note = (f"headroom ${headroom:.2f} ≈ {headroom / late_mean:.1f} typical late turns" + f" / {headroom / max_call:.1f} worst-case turns") + if headroom / max_call < wrapup_turns: + note += (f" — thinner than a {wrapup_turns}-turn worst case, so a hard stop" + " (`Curtailed(partial=None)`) mid-wrap-up is a TOLERATED outcome") + return note + + unobserved = [p for p in PHASES if phase_spend[p] == 0] + models = sorted({c.model for r in roots for c in r.calls}, key=str) + + lines: list[str] = [ + f"# Budget test matrix — calibrated from run `{run.run_id}`", + "", + f"Source window: {run.run['start_time']} .. {run.run['end_time']} ", + f"Models: {', '.join(map(str, models))} ", + f"Observed live LLM spend (1h cache-write bound): **${run_total:.2f}** ", + f"Warn threshold θ = {theta} (warn at θ·cap; hard stop is cooperative, at cost > cap)", + "", + "## Observed baselines (sub-agents folded into their root)", + "", + "| root | phase | calls | $ (5m..1h) |", + "|---|---|---:|---|", + *(f"| {r.description} | {r.phase} | {len(r.calls)} | " + f"${r.cost(False):.4f}..${r.cost():.4f} |" for r in roots), + "", + f"Author baseline: `{author.description}` — ${author_cost:.4f} over " + f"{len(author.calls)} calls (max ${max_call:.4f}, late-turn mean ${late_mean:.4f})." + + (" **Proxy**: no formalization-phase thread ran live in the source run (cache-warm);" + " the invariant author is the same agent shape." if author_is_proxy else ""), + ] + if unobserved: + lines += [ + "", + f"Phases with NO observed spend in the source run (cache-warm or subprocess): " + f"{', '.join(unobserved)} — their caps below are ample/uncalibrated. For a fully " + "calibrated matrix, source a cold-cache run.", + ] + + lines += [ + "", + "## Tests", + "", + "Run each against a **fresh `--cache-ns` and `--memory-ns`** (warm phases spend ~$0 " + "and can never trip a budget), e.g.:", + "", + " console-autoprove [system-doc] \\", + " --budget /t2_formalization_curtail.json \\", + " --cache-ns budget-t2-$(date +%s) --memory-ns budget-t2", + "", + f"### T1 — control (`t1_control.json`: total ${ample}, all caps ${ample})", + "Expected: identical behavior to an unbudgeted run. No wrap-up alerts in any " + "transcript, no `.unverified` files, report has no budget appendix, exit 0. Guards " + "against monitor-injection regressions.", + "", + f"### T2 — formalization curtailment (`t2_formalization_curtail.json`: " + f"formalization cap ${t2_cap})", + f"Warn fires at ≥ ${theta * t2_cap:.2f} of author spend " + f"({cap_fraction:.0%} of the observed ${author_cost:.2f} batch → warn lands " + f"~{theta * cap_fraction:.0%} of the way through). Expected: `Curtailed(partial)` — " + "wrap-up alert in the author transcript, no prover/judge calls after it, " + "`autospec_*.spec.unverified` on disk with no runnable sibling/conf, component in the " + f"report appendix, exit code reflects `all_failed` for single-component scenarios. " + f"{headroom_note(t2_cap)}.", + ] + if t3_cap is not None: + lines += [ + "", + f"### T3 — invariant curtailment (`t3_preparation_curtail.json`: " + f"formalization_preparation cap ${t3_cap})", + f"Warn fires at ≥ ${theta * t3_cap:.2f} into preparation spend (observed " + f"${prep_spend:.2f}). Expected: `invariants.spec.unverified`, NO invariant import " + "in component specs, 'Structural Invariants' appendix entry — and the run " + f"*continues* into component formalization. {headroom_note(t3_cap)}.", + ] + lines += [ + "", + f"### T4 — pool pressure (`t4_pool_pressure.json`: total ${t4_total}, caps ample)", + f"The shared pool warns at ≥ ${theta * t4_total:.2f} cumulative (observed full run " + f"${run_total:.2f}), so every agent from that point starts life in the wrap-up window. " + "Expected: one or more curtailed components, `all_failed` exit, extraction quietly " + "running fewer bug rounds under pressure (by design — observe, don't fail on it).", + "", + "### T5 — caching interplay (two runs, reuses `t2_formalization_curtail.json`)", + "Run A: T2 budget + fresh shared cache-ns → curtailed; verify via `cache-autoprove` " + "that NO generation cache entry exists for the curtailed component. Run B: same " + "cache-ns, `t1_control.json` → the author re-runs from scratch (no stale partial) and " + "delivers. Proves curtailed work is redone by a better-funded run.", + "", + "## Observables checklist (T2–T4)", + "", + "- exit code / `BUDGET:` lines in the failure summary", + "- `*.unverified` present; unsuffixed sibling absent; no conf for curtailed stems", + "- `report.json`: `curtailed_components` dispositions, coverage warning + count, " + "groups/`prover_links` exclude curtailed", + "- rendered HTML (`autoprove-report-render`) shows the budget appendix", + "- thread trail (`ap-trail export` + this script): the `` wrap-up " + "appears in the author transcript; no `verify_spec`/`feedback_tool` calls after it", + "- `components_to_prover_runs.json` lacks curtailed entries", + "", + "## Caveats", + "", + "- An unpriced model accrues $0 — budgets can NEVER trip on it. Check the models " + "line above against `composer/llm/pricing.py` before running.", + "- Calibration uses the 1h cache-write bound (authors run the long cache); the 5m " + "bound runs ~20% cheaper.", + "- AutoSetup's subprocess LLM calls never pass through `CostAccumulator`: " + "`formalization_preparation` meters only in-process work (invariants, summaries).", + "", + f"Regenerate: `uv run scripts/budget_math.py {run.run_id} --emit-matrix `", + "", + ] + return Matrix(files=files, manifest="\n".join(lines)) + + +# --------------------------------------------------------------------------- +# Reporting +# --------------------------------------------------------------------------- + +def _print_analysis(run: ExportedRun, roots: list[Root]) -> None: + print(f"run {run.run_id} ({run.run['start_time']} .. {run.run['end_time']})") + print(f"tags: {run.run['tags']}\n") + + hdr = (f"{'root (sub-agents folded)':52} {'phase':26} {'calls':>5} " + f"{'$5m':>8} {'$1h':>8}") + print(hdr) + print("-" * len(hdr)) + for r in roots: + print(f"{r.description[:52]:52} {r.phase:26} {len(r.calls):>5} " + f"{r.cost(False):>8.4f} {r.cost():>8.4f}") + for m in r.members: + if m != r.description: + print(f" + {m[:76]}") + + run5 = sum(r.cost(False) for r in roots) + run1 = sum(r.cost() for r in roots) + print(f"\nwhole-run LLM spend: ${run5:.4f} (all-5m) .. ${run1:.4f} (all-1h)") + + models: dict[str | None, int] = {} + unpriced: dict[str | None, int] = {} + for r in roots: + for c in r.calls: + models[c.model] = models.get(c.model, 0) + 1 + if not c.priced: + unpriced[c.model] = unpriced.get(c.model, 0) + 1 + print("models: " + ", ".join(f"{m}×{n}" for m, n in sorted(models.items(), key=lambda kv: -kv[1]))) + if unpriced: + print("\n!! UNPRICED MODELS (accrue $0 in production — budgets can NEVER trip on these):") + for m, n in unpriced.items(): + print(f" {m!r}: {n} call(s)") + + fat = max(roots, key=lambda r: r.cost()) + print(f"\npriciest root: {fat.description!r} — ${fat.cost():.4f} over {len(fat.calls)} calls") + running, marks = 0.0, {0.25, 0.5, 0.75} + total = fat.cost() or 1.0 + for n, c in enumerate((c.cost_1h for c in fat.calls), 1): + running += c + share = running / total + if (crossed := {m for m in marks if share >= m}): + marks -= crossed + print(f" call {n:>3}: ${running:.4f} ({share:.0%})") + + +def main() -> int: + curr_doc = __doc__ + assert curr_doc is not None + ap = argparse.ArgumentParser(description=curr_doc[0]) + ap.add_argument("source", help="run-trail source: a .json / .json.gz export, or a run id (fetched from the DB)") + ap.add_argument("--uid", default=None, + help="user id for DB fetch (defaults to $AUTOPROVER_USER_ID / _anonymous)") + ap.add_argument("--cap-fraction", type=float, default=0.75, + help="T2 curtailment cap as a fraction of the observed author-batch spend") + ap.add_argument("--wrapup-turns", type=int, default=4, + help="worst-case turns of headroom a graceful wrap-up wants") + ap.add_argument("--emit-matrix", type=Path, default=None, metavar="DIR", + help="write the calibrated budget files + manifest.md into DIR") + args = ap.parse_args() + + run = load_run(args.source, args.uid) + roots = _fold_roots(run) + if not any(r.calls for r in roots): + print("no priced LLM calls found in this run trail", file=sys.stderr) + return 1 + if run.threads and not any("cost_center" in t.meta for t in run.threads): + print( + "note: this trail predates cost-center tracking (no ThreadMeta.cost_center); " + "every root is unattributed and per-phase calibration is unavailable.", + file=sys.stderr, + ) + + _print_analysis(run, roots) + + if args.emit_matrix is not None: + matrix = build_matrix( + run, roots, cap_fraction=args.cap_fraction, wrapup_turns=args.wrapup_turns, + ) + outdir: Path = args.emit_matrix + outdir.mkdir(parents=True, exist_ok=True) + for name, payload in matrix.files.items(): + (outdir / name).write_text(json.dumps(payload, indent=2) + "\n") + (outdir / "manifest.md").write_text(matrix.manifest) + print(f"\nwrote {len(matrix.files)} budget file(s) + manifest.md -> {outdir}/") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/composer/scripts/rag_import.py b/composer/scripts/rag_import.py new file mode 100644 index 00000000..99558561 --- /dev/null +++ b/composer/scripts/rag_import.py @@ -0,0 +1,186 @@ +"""Generic RAG importer — ingest any corpus described by a common JSON manifest. + +This is the shared back half of the RAG build (see ``docs/rag-import-format.md``): it reads one or +more :class:`~composer.rag.import_format.RagManifest` documents and owns everything downstream of +the manifest — length-bounded chunking (``BlockBuilder``), embedding, ``part`` numbering, and the +DB writes. A manifest declares the two retrieval products separately, and each feeds exactly its +own index: + +* ``embedded_groups`` → ``add_chunks_batch`` — length-bounded embedded chunks for **vector** + (semantic) search, cut according to each block's declared kind; +* ``manual_sections`` → ``add_manual_section`` — whole documents for **keyword** search + exact + ``get_section``, never split. + +A *producer* does the corpus-specific parsing and emits the manifest; this module is +corpus-agnostic — an application ships its corpus as a committed ``.rag.json`` and +nothing about it lives here. + +Run under the ragbuild uv group (has spaCy + sentence-transformers):: + + uv run --isolated --group ragbuild python -m composer.scripts.rag_import \\ + corpus.rag.json [more.rag.json ...] [--output ] [--max-length N] [--print] +""" + +import argparse +import asyncio +import logging +import pathlib +from collections import defaultdict + +import spacy + +from composer.rag.db import KNOWLEDGE_BASES, get_rag_db +from composer.rag.import_format import ( + EmbeddedBlockKind, + EmbeddedGroup, + ManualBlockKind, + ManualSection, + RagManifest, + SCHEMA_VERSION, +) +from composer.rag.models import get_model +from composer.rag.text import code_ref_tag +from composer.rag.types import BlockChunk +from composer.scripts.text_processors import BlockBuilder, BuilderConfig + +logging.basicConfig(level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s") +logger = logging.getLogger(__name__) + +_BATCH_SIZE = 50 + + +def _manual_chunk(sec: ManualSection) -> BlockChunk: + """The whole section as one chunk (code as ```` tags) for keyword / get-section.""" + parts: list[str] = [] + code_refs: list[str] = [] + for b in sec.blocks: + if b.kind is ManualBlockKind.CODE: + parts.append(code_ref_tag(len(code_refs))) + code_refs.append(b.body) + else: + parts.append(b.body) + return BlockChunk(headers=list(sec.headers), part=0, code_refs=code_refs, chunk="\n\n".join(parts)) + + +def _embedded_chunks(group: EmbeddedGroup, config: BuilderConfig) -> list[BlockChunk]: + """Length-bounded embedded chunks for vector search, cut as each block's kind dictates.""" + builder = BlockBuilder(header=list(group.headers), config=config) + for b in group.blocks: + match b.kind: + case EmbeddedBlockKind.CODE: + builder.add_code(b.body) + case EmbeddedBlockKind.PARAGRAPH: + builder.append_text(b.body, is_structured_boundary=True, unbreakable=False) + case EmbeddedBlockKind.ATOMIC: + builder.append_text(b.body, is_structured_boundary=True, unbreakable=True) + case EmbeddedBlockKind.CONTINUATION: + builder.append_text(b.body, is_structured_boundary=False, unbreakable=False) + return list(builder.finish()) + + +def _load_manifest(path: pathlib.Path) -> RagManifest: + manifest = RagManifest.model_validate_json(path.read_text()) + if manifest.version != SCHEMA_VERSION: + raise SystemExit( + f"{path}: unsupported manifest version {manifest.version} (this importer speaks " + f"v{SCHEMA_VERSION}). Regenerate the manifest with a matching producer." + ) + return manifest + + +def _resolve_output(manifest: RagManifest, override: str | None) -> str: + if override: + return override + conn = KNOWLEDGE_BASES.get(manifest.knowledge_base) + if conn is None: + raise SystemExit( + f"no connection registered for knowledge_base {manifest.knowledge_base!r} " + f"(known: {sorted(KNOWLEDGE_BASES)}). Add it to composer.rag.db.KNOWLEDGE_BASES " + f"or pass --output ." + ) + return conn + + +def _print_manifest(manifest: RagManifest) -> None: + """Dry-run: render each manual section and name each embedded group, no DB writes.""" + print(f"=== knowledge_base: {manifest.knowledge_base} (source: {manifest.source})") + for s in manifest.manual_sections: + print(f"\n#### {' / '.join(h for h in s.headers if h)}") + print(_manual_chunk(s).chunk) + print(f"\n=== {len(manifest.embedded_groups)} embedded group(s):") + for g in manifest.embedded_groups: + kinds = ", ".join(b.kind.value for b in g.blocks) + print(f" {' / '.join(h for h in g.headers if h)} [{kinds}]") + + +async def _ingest( + db, manifest: RagManifest, config: BuilderConfig, seen_paths: dict[tuple[str, ...], int] +) -> tuple[int, int]: + """Ingest one manifest's two products into ``db``. ``seen_paths`` is shared across manifests + targeting the same DB so the ``manual_sections`` ``(headers, part)`` unique key never + collides.""" + buffer: list[BlockChunk] = [] + n_docs = 0 + for g in manifest.embedded_groups: + buffer.extend(_embedded_chunks(g, config)) + if len(buffer) >= _BATCH_SIZE: + await db.add_chunks_batch(buffer) + n_docs += len(buffer) + buffer = [] + if buffer: + await db.add_chunks_batch(buffer) + n_docs += len(buffer) + + for s in manifest.manual_sections: + manual = _manual_chunk(s) + key = tuple(manual.headers) + manual.part = seen_paths.get(key, 0) + seen_paths[key] = manual.part + 1 + await db.add_manual_section(manual) + return n_docs, len(manifest.manual_sections) + + +async def _async_main(args: argparse.Namespace) -> None: + manifests = [_load_manifest(f) for f in args.files] + + if args.print: + for m in manifests: + _print_manifest(m) + return + + config = BuilderConfig(nlp=spacy.load("en_core_web_sm"), max_length=args.max_length) + model = get_model() + + # Group by resolved target so manifests sharing a DB share one connection + one part counter. + groups: dict[str, list[RagManifest]] = defaultdict(list) + for m in manifests: + groups[_resolve_output(m, args.output)].append(m) + + for output, group in groups.items(): + db = await get_rag_db(output, model) + seen_paths: dict[tuple[str, ...], int] = {} + n_docs = n_manual = 0 + for m in group: + d, mn = await _ingest(db, m, config, seen_paths) + n_docs += d + n_manual += mn + logger.info( + "ingested %d embedded chunk(s) + %d manual section(s) from %d manifest(s) into %s", + n_docs, n_manual, len(group), output, + ) + + +def main() -> None: + parser = argparse.ArgumentParser(description="Ingest a RAG corpus from one or more JSON manifests.") + parser.add_argument("files", nargs="+", type=pathlib.Path, help="RAG manifest JSON files.") + parser.add_argument("--max-length", type=int, default=2000, help="Soft cap on embedded-chunk length (chars).") + parser.add_argument( + "--output", "-o", default=None, + help="RAG DB connection string. Overrides the manifest's knowledge_base -> connection lookup.", + ) + parser.add_argument("--print", action="store_true", help="Dry-run: print both products, no DB writes.") + asyncio.run(_async_main(parser.parse_args())) + + +if __name__ == "__main__": + main() diff --git a/composer/spec/artifacts.py b/composer/spec/artifacts.py index be90abc1..9aa8b284 100644 --- a/composer/spec/artifacts.py +++ b/composer/spec/artifacts.py @@ -18,13 +18,19 @@ from composer.diagnostics.timing import RunSummary from composer.spec.gen_types import PROPERTIES_SUBDIR, under_project -from composer.spec.types import PropertyFormulation +from composer.spec.types import CheckName, PropertyFormulation, PropertyTitle, VerificationArtifact from composer.spec.util import ensure_dir from .types import ArtifactIdentifier, FormalResult from composer.spec.source.report.schema import AutoProverReport _log = logging.getLogger(__name__) +#: Terminal suffix for artifacts persisted from a budget-curtailed generation. The +#: suffix keeps the file inert — no conf can reference a ``.spec.unverified`` and +#: forge won't compile a ``.t.sol.unverified`` — while stating exactly what's wrong +#: with it: the content never passed the validation gates. +QUARANTINE_SUFFIX = ".unverified" + class StoreConfiguration(TypedDict): internal_dir: Path | str @@ -60,8 +66,33 @@ def write_artifact(self, i: I, artifact: FormT) -> Path: self._write_commentary(i.stem, artifact.commentary) self._write_property_map( i.stem, self._property_suffix, - {k: v for (k,v) in artifact.property_units()}, + {k: v for (k,v) in artifact.property_checks()}, + ) + return target_path.relative_to(self._project_root) + + def write_quarantined(self, i: I, artifact: FormT) -> Path: + """Persist a budget-curtailed artifact for inspection under a poisoned name + (``{artifact_file}.unverified``). Only the artifact text is written — no + commentary, property map, or backend bundle — so nothing runnable or + machine-readable points at content that never passed the validation gates.""" + target_dir = ensure_dir(self._artifact_dir()) + target_path = target_dir / (i.artifact_file + QUARANTINE_SUFFIX) + target_path.write_text(artifact.artifact_text) + return target_path.relative_to(self._project_root) + + def write_plugin_artifact( + self, i: I, plugin: str, artifact: VerificationArtifact + ) -> Path: + """A verification-supporting artifact a plugin's tool registered for unit + ``i`` → ``{artifact_dir}/certificates/{stem}/{plugin}/{name}``. The name is + reduced to its basename and namespaced per unit and plugin, so tools cannot + traverse or collide. Returns the project-relative path (what the report + records).""" + target_dir = ensure_dir( + self._artifact_dir() / "certificates" / i.stem / plugin ) + target_path = target_dir / Path(artifact.name).name + target_path.write_text(artifact.content) return target_path.relative_to(self._project_root) def _deliverable_dir(self) -> Path: @@ -95,7 +126,7 @@ def _write_commentary(self, stem: str, commentary: str) -> None: (self._properties_dir() / f"{stem}.commentary.md").write_text(commentary) def _write_property_map( - self, stem: str, suffix: str, mapping: dict[str, list[str]], + self, stem: str, suffix: str, mapping: dict[PropertyTitle, list[CheckName]], ) -> None: """A ``{property title: [demonstrating names]}`` map → ``{stem}.{suffix}.json``. Titles are unique (enforced at extraction). ``suffix`` is the workflow's term diff --git a/composer/spec/code_explorer.py b/composer/spec/code_explorer.py index 201f52f7..cb456ae5 100644 --- a/composer/spec/code_explorer.py +++ b/composer/spec/code_explorer.py @@ -5,17 +5,17 @@ sub-agent with file system tools (list_files, get_file, grep_files). """ -from typing import NotRequired, override, Protocol, Any +from typing import Callable, Literal, NotRequired, override, Protocol, Any, TypedDict from pydantic import Field, BaseModel from langchain_core.tools import BaseTool from langgraph.graph.state import CompiledStateGraph -from langgraph.checkpoint.memory import InMemorySaver -from graphcore.graph import FlowInput, MessagesState +from graphcore.graph import FlowInput, MessagesState, TemplateLoader from graphcore.tools.schemas import WithAsyncImplementation, WithInjectedId +from composer.spec.gen_types import TypedTemplate from composer.spec.graph_builder import bind_standard, run_to_completion from composer.spec.tool_env import BaseSourceTools, BasicAgentTools from composer.spec.util import uniq_thread_id @@ -23,26 +23,27 @@ from composer.ui.tool_display import tool_display_of, CommonTools -CODE_EXPLORER_SYS_PROMPT = """\ -You are a code exploration assistant analyzing smart contract source code. -You have access to file tools (list_files, get_file, grep_files) to explore the project. +type PriorFindingsMode = Literal["none", "established", "versioned"] -Your job is to answer a specific question about the codebase thoroughly and precisely. -Guidelines: -- Ground every claim in what you find in the source code. -- Include relevant function signatures, state variable declarations, or code snippets in your answer. -- If the question asks about behavior, trace through the actual implementation rather than speculating. -- Be concise: the caller needs a dense, actionable answer, not a walkthrough of your exploration process. -- If you discover you do not have enough information to fully answer the question, - (e.g., there is a reference to code not available to you) *DO NOT GUESS*. Indicate in your final answer - that you cannot fully answer the question due to incomplete information. +class CodeExplorerPromptParams(TypedDict): + """Kwargs for an ecosystem's code-explorer system prompt. -If asked a question that cannot be answered by simply looking at the code (e.g., about some completely unrelated -topic) you must decline to answer, indicating it is out of scope for what you're capable of answering. + ``prior_findings`` selects which index protocol is appended: none (a + fresh explorer), established facts from the frozen index, or the + versioned/possibly-stale protocol used by the live editor. + """ + + prior_findings: PriorFindingsMode -When complete, deliver your answer via the `result` tool. -""" + +def code_explorer_sys_prompt( + template: TypedTemplate[CodeExplorerPromptParams], + prior_findings: PriorFindingsMode, +) -> Callable[[TemplateLoader], str]: + """The bound prompt as a deferred render, for `with_sys_prompt` to resolve + against the builder's own loader.""" + return template.bind({"prior_findings": prior_findings}).render_to class _ExplorerST(MessagesState): result: NotRequired[str] @@ -52,7 +53,7 @@ class CodeExplorerEnv(BaseSourceTools, BasicAgentTools, Protocol): def _code_explorer_graph( env: CodeExplorerEnv, - sys_prompt: str = CODE_EXPLORER_SYS_PROMPT + sys_prompt: Callable[[TemplateLoader], str], ) -> CompiledStateGraph[_ExplorerST, None, FlowInput, Any]: return bind_standard( env.builder, _ExplorerST, "Your findings about the source code" @@ -86,7 +87,7 @@ class _ExploreCodeCommon(BaseModel): * How to use other tools * Questions about CVL or the prover * Protocol related questions unrelated to the source code (e.g. expected deployment params, contract address seeds) -* Questions about the Solidity language itself +* Questions about the source language itself Good: 'What state variables does withdraw() modify and how?' Bad: 'Tell me about the contract' @@ -96,17 +97,24 @@ class _ExploreCodeCommon(BaseModel): ) -def code_explorer_tool(env: CodeExplorerEnv, recursion_limit: int) -> BaseTool: +def code_explorer_tool( + env: CodeExplorerEnv, + recursion_limit: int, + explorer_prompt: TypedTemplate[CodeExplorerPromptParams], +) -> BaseTool: """Create a code exploration sub-agent tool from a pre-configured builder. Args: env: Code explorer env with builder and tools bound. recursion_limit: LangGraph recursion limit for each sub-agent run. + explorer_prompt: Ecosystem-specific explorer system prompt. Returns: A BaseTool named ``explore_code``. """ - graph = _code_explorer_graph(env) + graph = _code_explorer_graph( + env, sys_prompt=code_explorer_sys_prompt(explorer_prompt, "none") + ) @tool_display_of(CommonTools.code_explorer) class ExploreCodeSchema(_ExploreCodeCommon, WithAsyncImplementation[str], WithInjectedId): @@ -138,18 +146,10 @@ def index(self) -> AgentIndex: def indexed_code_explorer_tool( env: ExtCodeExplorerEnv, recursion_limit: int, + explorer_prompt: TypedTemplate[CodeExplorerPromptParams], ) -> BaseTool: - - extended_sys = CODE_EXPLORER_SYS_PROMPT + f""" -You have access to findings from prior analyses of this codebase. -These findings were produced by earlier agents investigating the same contracts -and are established facts — do not re-derive or re-verify them. - -{AgentIndex.WITH_INDEX_SYS_COMMON} -""" - builder_graph = _code_explorer_graph( - env, sys_prompt=extended_sys + env, sys_prompt=code_explorer_sys_prompt(explorer_prompt, "established") ) @tool_display_of(CommonTools.code_explorer) diff --git a/composer/spec/context.py b/composer/spec/context.py index 8d02925e..0ba4295e 100644 --- a/composer/spec/context.py +++ b/composer/spec/context.py @@ -22,6 +22,7 @@ from composer.io.mnemonic_store import assign_mnemonic from composer.core.user import user_data_ns from composer.spec.types import SourceIdentifier +from composer.diagnostics.budget import budget_pressure # --------------------------------------------------------------------------- @@ -257,6 +258,11 @@ async def cache_get(self, ty: type[K]) -> K | None: async def cache_put(self, value: K) -> None: """Put a typed value in the cache. No-op if caching disabled.""" + if budget_pressure(): + # refuse to cache anything produced under budget pressure + # It's likely incomplete or "rushed", and a future run with more money + # should try again + return None if not isinstance(value, BaseModel): raise ValueError("Caching not allowed for non-basemodel keys") if self.cache_namespace is None: @@ -274,8 +280,3 @@ async def thread_and_mnemonic(self) -> tuple[str, str]: tid = self.thread_id mnem = await assign_mnemonic(tid, self._store, user_data_ns() + MNEMONIC_KEYS) return (tid, mnem) - -# --------------------------------------------------------------------------- -# Utility -# --------------------------------------------------------------------------- - diff --git a/composer/spec/cvl_generation.py b/composer/spec/cvl_generation.py index ca1c4877..7bc7dd36 100644 --- a/composer/spec/cvl_generation.py +++ b/composer/spec/cvl_generation.py @@ -6,12 +6,10 @@ - with_memory: whether to persist memory across runs """ -import hashlib from abc import ABC, abstractmethod from dataclasses import dataclass -from functools import cache -from typing import Annotated, Callable, Literal, NotRequired, Sequence, override, Awaitable, Any, Protocol -from typing_extensions import TypedDict +from typing import Callable, Literal, NotRequired, override, Awaitable, Any +from collections.abc import Sequence from pydantic import BaseModel, Field @@ -24,21 +22,20 @@ from graphcore.graph import FlowInput, tool_state_update, tool_return from graphcore.tools.schemas import WithInjectedState, WithInjectedId, WithAsyncDependencies +from composer.authoring.judge import PropertyFeedbackProtocol, RebuttalBase +from composer.authoring.state import ( + AuthoringExtra, MappingVocab, SkippedProperty, spec_digest, validate_check_mapping, +) +from composer.authoring.tools import skip_tools as _skip_pair from composer.spec.context import ( WorkflowContext, CacheKey, CVLGeneration, CVLJudge, ) from composer.spec.guidance import ERC20TokenGuidance, UnresolvedCallGuidance -from composer.core.state import merge_validation +from composer.spec.types import CheckName, PropertyTitle, RuleName from composer.spec.graph_builder import run_to_completion from composer.cvl.tools import put_cvl_raw, put_cvl, get_cvl, edit_cvl -from composer.ui.tool_display import tool_display, suppress_ack - -class PropertyFeedbackProtocol(Protocol): - @property - def good(self) -> bool: ... - - @property - def feedback(self) -> str: ... +from composer.ui.tool_display import tool_display +from composer.diagnostics.budget import budget_pressure CVL_JUDGE_KEY = CacheKey[CVLGeneration, CVLJudge]("judge") @@ -47,34 +44,10 @@ def feedback(self) -> str: ... # Feedback types # --------------------------------------------------------------------------- -class SkippedProperty(BaseModel): - """A property the agent explicitly decided not to formalize.""" - property_title: str = Field(description="The unique snake_case title of the property from the batch listing") - reason: str = Field(description="Justification for why this property was skipped") - - class PropertyRuleMapping(BaseModel): """The rules/invariants in the spec that verify a given property.""" - property_title: str = Field(description="The unique snake_case title of the property (from the batch listing) that these rules verify") - rules: list[str] = Field(description="The names of the rules/invariants in the spec that verify this property") - -class RebuttalBase(BaseModel): - prior_feedback_reference: str = Field( - description=( - "A brief quote from, or clear pointer to, the piece of prior-round feedback " - "this rebuttal addresses. Just enough for the judge to identify which prior " - "suggestion you are responding to — not a full transcript." - ) - ) - evidence: str = Field( - description=( - "The concrete artifact backing the rebuttal: typecheck error text, a " - "counterexample summary, a manual quote with location, or a brief reasoned " - "argument. Keep it short and specific — the judge reads this verbatim." - ) - ) - - + property_title: PropertyTitle = Field(description="The unique snake_case title of the property (from the batch listing) that these rules verify") + rules: list[RuleName] = Field(description="The names of the rules/invariants in the spec that verify this property") class Rebuttal(RebuttalBase): """A rebuttal to a specific piece of feedback from a prior round, backed by evidence. @@ -99,24 +72,6 @@ class Rebuttal(RebuttalBase): ) -def _merge_skips( - left: list[SkippedProperty], - right: list[SkippedProperty], -) -> list[SkippedProperty]: - """State reducer: merge by property_title (new justification replaces old). - - An entry with an empty reason is a sentinel for "unskipped" — it removes - the property from the skip list. - """ - by_title = {s.property_title: s for s in left} - for s in right: - by_title[s.property_title] = s - return sorted( - (s for s in by_title.values() if s.reason), - key=lambda s: s.property_title, - ) - - def _output_link(link: str | None) -> str | None: """Rewrite a prover ``/jobStatus/`` URL to its ``/output/`` view; local result dirs (and ``None``) pass through unchanged.""" @@ -149,8 +104,8 @@ class GeneratedCVL(BaseModel): vfs: dict[str, str] = Field(default_factory=dict) applied_edits: list[AppliedEdit] = Field(default_factory=list) - def property_units(self) -> list[tuple[str, list[str]]]: - """Property title -> the CVL rule names that formalize it (the report's `ReportableResult` + def property_checks(self) -> list[tuple[PropertyTitle, list[RuleName]]]: + """Property title -> the CVL rule names that verify it (the report's `ReportableResult` adapter; pairs with the structurally-shared ``skipped`` field).""" return [(m.property_title, m.rules) for m in self.property_rules] @@ -169,110 +124,29 @@ def output_link(self) -> str | None: # Completion validation # --------------------------------------------------------------------------- -class CVLGenerationExtra(TypedDict): - curr_spec: str | None - skipped: Annotated[list[SkippedProperty], _merge_skips] +class CVLGenerationExtra(AuthoringExtra): property_rules: list[PropertyRuleMapping] - validations: Annotated[dict[str, str], merge_validation] - required_validations: list[str] -def _compute_digest( - curr_spec: str, - skipped: list[SkippedProperty], - version_history: Sequence[str] = (), -) -> str: - """Digest of everything a validation stamp vouches for: the spec, the skip - set, and — in the editing-enabled source pipeline — the applied-edit - history, so a stamp earned before a source edit goes stale with it. - Pipelines without source editing pass no history and hash identically to - before.""" - digester = hashlib.md5() - digester.update(curr_spec.encode()) - for s in skipped: - digester.update(f"{s.property_title}:{s.reason}".encode()) - for edit_id in version_history: - digester.update(f"edit:{edit_id}".encode()) - return digester.hexdigest() - - -def check_completion( - state: CVLGenerationExtra, - version_history: Sequence[str] = (), -) -> str | None: - """Returns None if valid, error string if not.""" - spec = state["curr_spec"] - if spec is None: - return "Completion REJECTED: no spec written yet." - digest = _compute_digest(spec, state["skipped"], version_history) - validations = state["validations"] - required = state["required_validations"] - for key in required: - if key not in validations or validations[key] != digest: - return f"Completion REJECTED: {key} validation not satisfied or stale." - return None +#: How the CVL author words its publish-time mapping. The prover reports no rule-name ground truth +#: (unlike forge), so ``validate_property_rules`` passes no ``ran`` set and the mapping is checked +#: for coverage only. +_CVL_MAPPING = MappingVocab(check_noun="rule", field_name="property_rules") def validate_property_rules( property_rules: list[PropertyRuleMapping], skipped: list[SkippedProperty], - titles: list[str], + titles: list[PropertyTitle], + known_rules: set[str] | None = None ) -> str | None: - """Validate the property->rules mapping declared at completion time. - - ``titles`` is the batch's full set of property titles. Returns None if valid, otherwise - a single message enumerating all problems. A mapping is valid when every non-skipped - property (referenced by its unique title) is mapped to at least one rule, no skipped - property is mapped, every referenced title exists, and no title is mapped twice. - """ - valid_titles = set(titles) - skipped_titles = {s.property_title for s in skipped} - errors: list[str] = [] - mapped: set[str] = set() - for m in property_rules: - if m.property_title not in valid_titles: - errors.append(f"Unknown property title {m.property_title!r} (not one of the batch's properties).") - continue - if m.property_title in mapped: - errors.append(f"Property {m.property_title!r} appears more than once in the mapping.") - continue - mapped.add(m.property_title) - if m.property_title in skipped_titles: - errors.append( - f"Property {m.property_title!r} is marked as skipped and must not appear " - "in the mapping (un-skip it or remove it)." - ) - continue - if not any(r.strip() for r in m.rules): - errors.append(f"Property {m.property_title!r} must map to at least one non-empty rule name.") - for t in titles: - if t in skipped_titles or t in mapped: - continue - errors.append(f"Property {t!r} is neither skipped nor mapped to any rules.") - if errors: - return ( - "Completion REJECTED: the property_rules mapping is invalid. Fix all of the " - "following and resubmit:\n- " + "\n- ".join(errors) - ) - return None - - -def make_validation_stamper( - key: str, -) -> Callable[[CVLGenerationExtra, Sequence[str]], dict[str, str]]: - """Create a stamper for future prover tool integration. - - The stamper reads curr_spec/skipped from state — plus the caller's - applied-edit history — and returns a dict suitable for merging into the - validations state key. - """ - def stamp(state: CVLGenerationExtra, version_history: Sequence[str]) -> dict[str, str]: - return {key: _compute_digest( - state["curr_spec"] or "", - state["skipped"], - version_history, - )} - return stamp + """Validate the property->rules mapping declared at completion time. ``titles`` is the batch's + full set of property titles; returns None if valid, else one message enumerating all problems.""" + return validate_check_mapping( + [(m.property_title, m.rules) for m in property_rules], skipped, titles, _CVL_MAPPING, ran=[ + CheckName(it) for it in known_rules + ] if known_rules else None + ) class CVLGenerationInput(FlowInput, CVLGenerationExtra): @@ -308,7 +182,7 @@ class FeedbackServices: # The batch's property titles (unique, enforced at extraction). Used to validate that # the titles named by record_skip / unskip_property / the result mapping refer to real # properties, and to check every non-skipped property is mapped. - titles: list[str] + titles: list[PropertyTitle] FEEDBACK_VALIDATION_KEY = "feedback" @@ -355,11 +229,21 @@ async def run(self) -> Command: spec = st["curr_spec"] if spec is None: return tool_return(self.tool_call_id, "No spec put yet") + if budget_pressure(): + # Don't launch a judge that would be terminated on its first + # monitor tick; the author's budget warning already tells it + # feedback approval is no longer required. + return tool_return( + self.tool_call_id, + "Good? False\nFeedback The feedback judge was not run due to budget " + "constraints. See the system alert: feedback approval is no longer " + "required for this task.", + ) skipped = st["skipped"] t = await self._get_feedback(spec, skipped) msg = f"Good? {t.good}\nFeedback {t.feedback}" if t.good: - digest = _compute_digest(spec, skipped, self._version_history()) + digest = spec_digest(spec, skipped, self._version_history()) return tool_state_update( self.tool_call_id, msg, validations={FEEDBACK_VALIDATION_KEY: digest}, @@ -385,79 +269,6 @@ async def _get_feedback( def _version_history(self) -> Sequence[str]: return () -@tool_display( - lambda p: f"Skipping property `{p.get('property_title', '?')}`", - suppress_ack("Skip result", ("Recorded skip",)), -) -class _RecordSkipSchema(WithInjectedId, WithAsyncDependencies[Command, list[str]]): - """ - Declare that you are skipping a property from the batch. - You must provide the property's title and a justification. - The feedback judge will evaluate whether your justification is valid. - Only use this after genuinely attempting to formalize the property. - """ - property_title: str = Field( - description="The snake_case title of the property from the batch listing" - ) - reason: str = Field( - description="Justification for why this property cannot be formalized" - ) - - @override - async def run(self) -> Command: - with self.tool_deps() as titles: - if self.property_title not in titles: - return tool_state_update( - self.tool_call_id, - f"Unknown property title {self.property_title!r}. Must be one of: {', '.join(titles)}.", - ) - if not self.reason.strip(): - return tool_state_update( - self.tool_call_id, - "A non-empty justification is required when skipping a property.", - ) - skip = SkippedProperty( - property_title=self.property_title, - reason=self.reason, - ) - return tool_state_update( - self.tool_call_id, - f"Recorded skip for property {self.property_title}.", - skipped=[skip], - ) - -@tool_display( - lambda p: f"Un-skipping property `{p.get('property_title', '?')}`", - suppress_ack("Unskip result", ("Removed skip",)), -) -class _UnskipSchema(WithInjectedId, WithAsyncDependencies[Command, list[str]]): - """ - Remove a previously declared skip for a property. - Use this if you later find a way to formalize a property you previously skipped. - """ - property_title: str = Field( - description="The snake_case title of the property to un-skip" - ) - - @override - async def run(self) -> Command: - with self.tool_deps() as titles: - if self.property_title not in titles: - return tool_state_update( - self.tool_call_id, - f"Unknown property title {self.property_title!r}. Must be one of: {', '.join(titles)}.", - ) - # Empty reason is the sentinel for "not skipped" - skip = SkippedProperty( - property_title=self.property_title, - reason="", - ) - return tool_state_update( - self.tool_call_id, - f"Removed skip for property {self.property_title}.", - skipped=[skip], - ) - def static_tools() -> list[BaseTool]: """The dependency-free CVL authoring tools. The property-management suite (feedback / skip tools) is NOT here — it carries runtime deps; see @@ -471,12 +282,23 @@ def static_tools() -> list[BaseTool]: ] -def skip_tools(titles: list[str]) -> list[BaseTool]: +_SKIP_DESCRIPTION = """ + Declare that you are skipping a property from the batch. + You must provide the property's title and a justification. + The feedback judge will evaluate whether your justification is valid. + Only use this after genuinely attempting to formalize the property. + """ + +_SKIP_REASON = "Justification for why this property cannot be formalized" + + +def skip_tools(titles: list[PropertyTitle]) -> list[BaseTool]: """The skip-management pair, bound to the batch's property titles.""" - return [ - _RecordSkipSchema.bind(titles).as_tool("record_skip"), - _UnskipSchema.bind(titles).as_tool("unskip_property"), - ] + return _skip_pair( + titles, + skip_description=_SKIP_DESCRIPTION, + skip_reason=_SKIP_REASON, + ) def property_tools(services: FeedbackServices) -> list[BaseTool]: diff --git a/composer/spec/feedback.py b/composer/spec/feedback.py index 00dd1e1c..0b3d25fc 100644 --- a/composer/spec/feedback.py +++ b/composer/spec/feedback.py @@ -1,40 +1,29 @@ import inspect from dataclasses import dataclass -from typing import Callable, NotRequired, Protocol, Sequence, Awaitable +from typing import Awaitable, Callable, NotRequired, Sequence from typing_extensions import TypedDict from composer.spec.service_host import Sort, ServiceHost -from pydantic import BaseModel, Field - -from langchain_core.tools import BaseTool -from langgraph.graph import MessagesState - -from graphcore.graph import Builder, FlowInput +from graphcore.graph import Builder from graphcore.tools.vfs import VFSState +from composer.authoring.judge import ( + JudgeInput, JudgeState, JudgeToolHost, ContextualFeedbackThunk, + build_feedback_judge_generic, judge_host_of, +) +from composer.authoring.state import SkippedProperty from composer.spec.context import ( WorkflowContext, CVLJudge ) from composer.spec.types import PropertyFormulation -from composer.spec.graph_builder import bind_standard, run_to_completion from composer.cvl.tools import get_cvl -from composer.tools.thinking import RoughDraftState, get_rough_draft_tools -from composer.spec.cvl_generation import FeedbackServices, Rebuttal, SkippedProperty -from composer.spec.source.live_explorer import VersionedHistory -from composer.spec.system_model import ContractComponentInstance from composer.spec.gen_types import TemplateInstantiation, TypedTemplate, ITypedTemplate, PartialTemplate +from composer.spec.cvl_generation import FeedbackServices, Rebuttal +from composer.spec.source.live_explorer import VersionedHistory from composer.spec.system_model import ContractComponentInstance, component_context -from composer.spec.util import uniq_thread_id from composer.kb.kb_context import with_cvl_context -class PropertyFeedback(BaseModel): - """ - The feedback on the properties - """ - good: bool = Field(description="Whether the properties are good as is, or if there is room for improvement") - feedback: str = Field(description="The feedback on the rule if work is needed. Can be empty if there is no feedback") - class Properties(TypedDict): properties: list[PropertyFormulation] @@ -74,16 +63,6 @@ class JudgeSystemParams(TypedDict): FeedbackSystemTemplate = TypedTemplate[JudgeSystemParams]("property_judge_system_prompt.j2") -class JudgeExtra(RoughDraftState): - curr_spec: str - - -class FeedbackBaseState(MessagesState, JudgeExtra): - result: NotRequired[PropertyFeedback] - -class FeedbackBaseInput(FlowInput, JudgeExtra): - pass - # Extra input parts prepended to every judge invocation. A bare list is static; # a callable is evaluated per invocation (and may be async) so the producer can # reflect state that changes between review rounds — e.g. a notice that the @@ -95,133 +74,72 @@ class FeedbackBaseInput(FlowInput, JudgeExtra): | None ) -type ContextualFeedbackToolImpl[Ctx] = Callable[ - [Ctx, str, list[SkippedProperty], list[Rebuttal], str], - Awaitable[PropertyFeedback] -] - - -class JudgeToolHost(Protocol): - """The judge's construction surface: a builder, the workflow ``sort``, and - the tool suite the judge runs with. Callers vary the FS-read strategy - through ``judge_tools`` — frozen fs tools over the project root, or - vfs-aware tools reading the author's working copy — without the judge - machinery knowing which. ``ServiceHost`` consumers adapt via - :func:`judge_host_of`.""" - - def builder_heavy(self) -> Builder[None, None, None]: ... - - @property - def sort(self) -> Sort: ... - - @property - def judge_tools(self) -> tuple[BaseTool, ...]: ... - - -@dataclass(frozen=True) -class _ServiceHostJudge: - """The vanilla adapter: judge runs with the host's full tool surface.""" - env: ServiceHost - - def builder_heavy(self) -> Builder[None, None, None]: - return self.env.builder_heavy() - - @property - def sort(self) -> Sort: - return self.env.sort - - @property - def judge_tools(self) -> tuple[BaseTool, ...]: - return self.env.all_tools - +type ContextualFeedbackToolImpl[Ctx] = ContextualFeedbackThunk[Rebuttal, Ctx] -def judge_host_of(env: ServiceHost) -> JudgeToolHost: - return _ServiceHostJudge(env) - -def property_feedback_judge_generic[ - S: FeedbackBaseState, - I: FeedbackBaseInput, - Ctx -]( +def property_feedback_judge_generic[S: JudgeState, I: JudgeInput, Ctx]( st: type[S], - i: type[I], + inp: type[I], ctx: WorkflowContext[CVLJudge], host: JudgeToolHost, prompt: ITypedTemplate[FeedbackInputs], props: list[PropertyFormulation], - extra_inputs: ExtraInputPrompt, system_prompt: TemplateInstantiation | None, - - input_lift: Callable[[FeedbackBaseInput, Ctx], I], + input_lift: Callable[[JudgeInput, Ctx], I], source_editing: bool = False, ) -> ContextualFeedbackToolImpl[Ctx]: + """The CVL property judge, generic over its state/input pair and a per-invocation context. + + The batch's properties, skips and rebuttals are all rendered into the judge's *prompt* here + (``property_judge_prompt.j2``); only the spec under review and the caller's ``extra_inputs`` + ride in as input text.""" if system_prompt is None: system_prompt = FeedbackSystemTemplate.bind( {"sort": host.sort, "source_editing": source_editing} ) - - builder = host.builder_heavy().with_tools( - host.judge_tools - ) - - rough_draft_tools = get_rough_draft_tools(st) - - def did_rough_draft_read(s: S, _) -> str | None: - if not s["did_read"]: - return "Completion REJECTED: never read rough draft for review" - return None - - mem = ctx.get_memory_tool() - - staged_workflow = bind_standard( - builder, st, validator=did_rough_draft_read - ).with_input( - i - ).inject( - lambda g: system_prompt.render_to(g.with_sys_prompt_template) - ).with_tools([*rough_draft_tools, mem, get_cvl(st), ]) - - async def the_tool( - exec_ctx: Ctx, - cvl: str, - skipped: Sequence[SkippedProperty], - rebuttals: Sequence[Rebuttal], - within_tool: str, - ) -> PropertyFeedback: - workflow = staged_workflow.with_initial_prompt( - with_cvl_context(prompt.bind({ - "properties": props, - "rebuttals": rebuttals, - "skipped": skipped - }).render_to) - ).compile_async() - - input_parts: list[str | dict] = [] + bound_system = system_prompt + + def apply_prompt( + builder: Builder[S, None, I], _cvl: str, + skipped: Sequence[SkippedProperty], rebuttals: Sequence[Rebuttal], + ) -> Builder[S, None, I]: + return builder.with_initial_prompt(with_cvl_context(prompt.bind({ + "properties": props, + "rebuttals": rebuttals, + "skipped": skipped, + }).render_to)) + + async def input_parts( + cvl: str, _skipped: Sequence[SkippedProperty], _rebuttals: Sequence[Rebuttal] + ) -> list[str | dict]: + parts: list[str | dict] = [] if extra_inputs: if isinstance(extra_inputs, list): - input_parts.extend(extra_inputs) + parts.extend(extra_inputs) else: produced = extra_inputs() if inspect.isawaitable(produced): produced = await produced - input_parts.extend(produced) - - input_parts.append("The proposed CVL file is") - input_parts.append(cvl) - res = await run_to_completion( - workflow, - input_lift(FeedbackBaseInput(input=input_parts, curr_spec=cvl, memory=None, did_read=False), exec_ctx), - thread_id=uniq_thread_id("feedback"), - recursion_limit=ctx.recursion_limit, - description="Property feedback judge", - within_tool=within_tool, - ) - assert "result" in res - return res["result"] - return the_tool + parts.extend(produced) + parts.append("The proposed CVL file is") + parts.append(cvl) + return parts + + return build_feedback_judge_generic( + st=st, + inp=inp, + ctx=ctx, + host=host, + apply_system=lambda b: bound_system.render_to(b.with_sys_prompt_template), + apply_prompt=apply_prompt, + input_parts=input_parts, + readback=get_cvl(st), + description="Property feedback judge", + thread_prefix="feedback", + input_lift=input_lift, + ) def property_feedback_judge( @@ -236,16 +154,19 @@ def property_feedback_judge( """The vanilla judge: the full ``ServiceHost`` tool surface (frozen FS reads), no source editing. Returns the services bundle the property- management tool suite binds against.""" + def lift(i: JudgeInput, _: None) -> JudgeInput: + return i + to_wrap = property_feedback_judge_generic( - st=FeedbackBaseState, - i=FeedbackBaseInput, + st=JudgeState, + inp=JudgeInput, ctx=ctx, host=judge_host_of(env), extra_inputs=extra_inputs, prompt=prompt, props=props, system_prompt=system_prompt, - input_lift=lambda i, _: i, + input_lift=lift, ) return FeedbackServices( @@ -268,11 +189,11 @@ class SourceSnapshot: version_history: list[str] -class VfsJudgeState(FeedbackBaseState, VFSState, VersionedHistory): +class VfsJudgeState(JudgeState, VFSState, VersionedHistory): pass -class VfsJudgeInput(FeedbackBaseInput, VFSState, VersionedHistory): +class VfsJudgeInput(JudgeInput, VFSState, VersionedHistory): pass @@ -290,7 +211,7 @@ def source_feedback_judge( :class:`SourceSnapshot`); the returned impl takes that snapshot as its leading argument on every invocation.""" - def lift(base: FeedbackBaseInput, snap: SourceSnapshot) -> VfsJudgeInput: + def lift(base: JudgeInput, snap: SourceSnapshot) -> VfsJudgeInput: return VfsJudgeInput( **base, vfs=snap.vfs, @@ -299,7 +220,7 @@ def lift(base: FeedbackBaseInput, snap: SourceSnapshot) -> VfsJudgeInput: return property_feedback_judge_generic( st=VfsJudgeState, - i=VfsJudgeInput, + inp=VfsJudgeInput, ctx=ctx, host=host, extra_inputs=extra_inputs, diff --git a/composer/spec/graph_builder.py b/composer/spec/graph_builder.py index 5f3bb239..0fcaa4d0 100644 --- a/composer/spec/graph_builder.py +++ b/composer/spec/graph_builder.py @@ -2,22 +2,22 @@ Convenience helpers for building agent sub-workflows. - bind_standard: Extracts result type from state, adds result tool + summarizer -- run_to_completion: Thin wrapper around context.run_graph for sub-workflows +- run_to_completion: re-exported from ``composer.io.context`` (its real home), + where retry policies are applied; kept here for the existing spec-land + importers. """ from typing import Any, Callable, NotRequired, get_origin, get_args, cast, overload from pydantic import BaseModel -from langchain_core.runnables import RunnableConfig from langgraph._internal._typing import StateLike from langgraph.graph import MessagesState -from langgraph.graph.state import CompiledStateGraph from graphcore.graph import Builder, FlowInput from graphcore.tools.results import ValidationResult, result_tool_generator -from composer.io.context import run_graph as _context_run_graph +from composer.io.context import run_to_completion as run_to_completion def bind_standard[_S: MessagesState, _C: StateLike | None, _I: FlowInput | None, _R]( @@ -73,41 +73,3 @@ def bind_standard[_S: MessagesState, _C: StateLike | None, _I: FlowInput | None, return builder.with_state(state_type).with_tools([result_tool]).with_output_key("result").with_default_summarizer() -async def run_to_completion[I: StateLike, S: StateLike, C: StateLike | None]( - graph: CompiledStateGraph[S, C, I, Any], - input: I, - thread_id: str, - context: C = None, - *, - checkpoint_id: str | None = None, - recursion_limit: int, - description: str, - within_tool: str | None = None, -) -> S: - """Run a compiled state graph to completion. - - Delegates to composer.io.context.run_graph, which handles event nesting - automatically via context vars. Requires with_handler() to be active. - - ``within_tool`` is the calling tool's ``tool_call_id`` when this graph is - being run as a sub-agent from inside a tool. It anchors the sub-graph's - UI panel under the tool-call widget so the renderer can mount nested - output in the right place. Pass ``self.tool_call_id`` from a tool that - mixes in ``WithInjectedId``; leave ``None`` for top-level / pipeline- - phase invocations. - """ - run_conf: RunnableConfig = { - "configurable": {"thread_id": thread_id}, - "recursion_limit": recursion_limit, - } - if checkpoint_id is not None: - run_conf["configurable"]["checkpoint_id"] = checkpoint_id - - return await _context_run_graph( - graph=graph, - ctxt=context, - input=input, - run_conf=run_conf, - description=description, - within_tool=within_tool, - ) diff --git a/composer/spec/key_family.py b/composer/spec/key_family.py new file mode 100644 index 00000000..910fe153 --- /dev/null +++ b/composer/spec/key_family.py @@ -0,0 +1,87 @@ +"""Declared cache-key derivation rules. + +``WorkflowContext.child`` walks the cache tree one typed edge at a time: +a ``CacheKey[Parent, Child]`` names the child slot and carries the +(phantom) evidence that the transition is legal. The key *strings*, +though, were historically minted by ad-hoc helpers — file-local +functions gluing digests together with f-strings, each returning a bare +``CacheKey(...)`` whose type arguments were whatever the annotation +happened to claim. Nothing tied a derivation rule to the edge it +produces, and consumers that re-derive keys (the cache explorers) had +to import private helpers from whichever module each rule happened to +live in. + +A :class:`KeyFamily` makes the rule the declared object: one value +pinning the parent type, the child type, and the derivation from the +parameters that identify a member of the family. Calling the family is +the only way to mint that edge's key, so every producer and every +explorer agree on the string and the types by construction. + +Conventions: + +- Families are declared next to their cache models (usually the module + of the producing agent) and named ``*_KEY``, like the constant + ``CacheKey`` slots they generalize. +- Each application gathers its families into a registry module — + ``composer.pipeline.keys`` for the Auto-Prove driver chain, + ``composer.spec.source.keys`` for the prover backend's sub-chain — so + one file shows the whole cache tree. Registries import producers, + never the reverse. +""" + +from dataclasses import dataclass +from typing import Callable + +from composer.spec.context import CacheKey + + +@dataclass(frozen=True) +class KeyFamily[Parent, Child, **P]: + """The derivation rule for a ``Parent → Child`` cache edge. + + ``parent`` / ``child`` are runtime witnesses that bind the type + parameters (``CacheKey``'s are phantom, so nothing else could); + ``derive`` maps the values identifying one member of the family to + its key string. Calling the family yields the typed key:: + + AUTOSETUP_KEY = KeyFamily(ContractSetup, SetupSuccess, _autosetup_key) + ... + ctx.child(AUTOSETUP_KEY(app, prover_opts)) + + A fixed edge (no parameters) is just a ``CacheKey`` constant; declare + a family only when there is something to derive. + """ + + parent: type[Parent] + child: type[Child] + derive: Callable[P, str] + + def __call__(self, *args: P.args, **kwargs: P.kwargs) -> CacheKey[Parent, Child]: + return CacheKey(self.derive(*args, **kwargs)) + + +@dataclass(frozen=True) +class PolyKeyFamily[Parent, **P]: + """A :class:`KeyFamily` whose child type is a call-site parameter. + + For the edges the driver traverses generically — the formalization + result is the *backend's* result type, the analyzed system model is + the *ecosystem's* — no static declaration can name the child. The + caller passes the child's runtime witness instead, which is exactly + what those call sites already hold (``formalizer.formalized_type``, + ``ecosystem.system_model``), and inference does the rest — no + ``CacheKey[...]`` respelling, no annotation-steered inference of an + unparameterized call:: + + child_ctx = await ctx.child( + FORMALIZATION_KEY(formalizer.formalized_type, batch.props) + ) + """ + + parent: type[Parent] + derive: Callable[P, str] + + def __call__[Child]( + self, child: type[Child], /, *args: P.args, **kwargs: P.kwargs + ) -> CacheKey[Parent, Child]: + return CacheKey(self.derive(*args, **kwargs)) diff --git a/composer/spec/natspec/author.py b/composer/spec/natspec/author.py index 43ed59fe..ddb66228 100644 --- a/composer/spec/natspec/author.py +++ b/composer/spec/natspec/author.py @@ -10,10 +10,12 @@ from graphcore.summary import SummaryConfig from graphcore.graph import tool_state_update -from graphcore.tools.schemas import WithImplementation, WithInjectedId, WithInjectedState, WithAsyncDependencies +from graphcore.tools.schemas import WithInjectedId, WithInjectedState, WithAsyncDependencies +from composer.authoring.state import check_completion +from composer.authoring.tools import give_up_tool from composer.spec.cvl_generation import ( static_tools, property_tools, run_cvl_generator, CVLGenerationInput, CVLGenerationState, - check_completion, CVLGenerationExtra + CVLGenerationExtra ) from composer.spec.service_host import Sort @@ -24,7 +26,8 @@ from composer.spec.feedback import property_feedback_judge, Properties, FeedbackTemplate from composer.spec.gen_types import TypedTemplate from composer.spec.system_model import ContractComponentInstance, ContractName, component_context -from composer.spec.cvl_generation import CVL_JUDGE_KEY, static_tools, SkippedProperty +from composer.authoring.state import SkippedProperty +from composer.spec.cvl_generation import CVL_JUDGE_KEY from composer.spec.service_host import ServiceHost from composer.kb.kb_context import with_cvl_context from composer.ui.tool_display import tool_display, suppress_ack @@ -109,27 +112,12 @@ class GaveUp(BaseModel): class AuthorResult(BaseModel): result_wrapped: Annotated[GaveUp | GenerationSuccess, Discriminator("ty")] -@tool_display( - label=lambda p: f"Giving up on property generation: {p['reason']}", - result=None, -) -class GiveUpTool(WithImplementation[Command], WithInjectedId): - """ +_GIVE_UP_DESCRIPTION = """ Call this tool to give up on the property generation for this task. This should only ever be called as a LAST RESORT when you have exhausted all other mechanisms to complete your task """ - reason : str = Field(description="The reason for giving up on your task") - - @override - def run(self) -> Command: - return tool_state_update( - self.tool_call_id, - "Accepted", - failed=True, - result=self.reason - ) @dataclass class ContractConfiguration: @@ -213,9 +201,9 @@ async def generate_cvl_batch( def stub_feedback_extras() -> list[str | dict]: return [ - f"The current typechecking stub for the {contract_name} contract is", + f"The current typechecking stub for the {contract_name} contract is\n\n", stub_reader(), - "For reference, the system document for the application is", + "For reference, the system document for the application is\n\n", system_doc.content.to_dict(), ] @@ -235,7 +223,7 @@ def stub_feedback_extras() -> list[str | dict]: .with_tools(static_tools()) .with_tools(property_tools(feedback_services)) .with_tools([ - GiveUpTool.as_tool("give_up"), + give_up_tool(name="give_up", description=_GIVE_UP_DESCRIPTION, label="property generation"), AdvisoryTypecheck.bind(typechecker).as_tool("advisory_typecheck"), PublishTool.bind(typechecker).as_tool("publish"), ctx.get_memory_tool() diff --git a/composer/spec/natspec/pipeline.py b/composer/spec/natspec/pipeline.py index f9a496c2..da353913 100644 --- a/composer/spec/natspec/pipeline.py +++ b/composer/spec/natspec/pipeline.py @@ -244,7 +244,7 @@ async def _analyze_component(component_idx: int) -> _ComponentBatch | None: extra_input=[ CacheablePropertyGenerationInput( "certora:system-doc", "generic", "always", lambda cache: [ - "For reference, the system document describing the entire application is as follows.", + "For reference, the system document describing the entire application is as follows.\n\n", system_doc.content.to_dict(CacheLevel.SHORT if cache else CacheLevel.NONE) ] ) diff --git a/composer/spec/prop_inference.py b/composer/spec/prop_inference.py index 0f94cc1a..e137a501 100644 --- a/composer/spec/prop_inference.py +++ b/composer/spec/prop_inference.py @@ -16,7 +16,9 @@ from composer.input.files import Document from composer.llm.types import CacheLevel from composer.spec.gen_types import TypedTemplate +from composer.spec.util import combine_digests from composer.spec.context import WorkflowContext, CacheKey, ComponentGroup +from composer.spec.key_family import KeyFamily from composer.spec.gen_types import TypedTemplate from composer.spec.graph_builder import bind_standard, run_to_completion from composer.spec.types import PropertyFormulation @@ -24,6 +26,7 @@ from composer.tools.thinking import RoughDraftState, get_rough_draft_tools from composer.spec.service_host import Sort, ServiceHost from composer.io.conversation import ConversationContextProvider +from composer.diagnostics.budget import budget_monitor, budget_pressure from composer.templates.loader import load_jinja_template from composer.spec.prop_refinement import user_property_refinement @@ -110,33 +113,33 @@ def render_evm_property_prompt( class _AgentRoundWithHistory(_AgentRoundResult): agent_conversation: list[AnyMessage] -def bug_analysis_key_from_digest( +def _bug_analysis_key( threat_model_digest: str | None, - with_refinement: bool -) -> CacheKey[ComponentGroup, _BugAnalysisCache]: + with_refinement: bool, + extra_context_digest: str | None = None, +) -> str: base_key = "bug_analysis" if with_refinement: base_key += "|refine" - if threat_model_digest is None: - return CacheKey[ComponentGroup, _BugAnalysisCache](base_key) - return CacheKey[ComponentGroup, _BugAnalysisCache](base_key + "-tm-" + threat_model_digest) - -def bug_analysis_key( - threat_model: Document | None, - with_refinement: bool -) -> CacheKey[ComponentGroup, _BugAnalysisCache]: - return bug_analysis_key_from_digest( - threat_model.to_digest() if threat_model is not None else None, - with_refinement, - ) + if threat_model_digest is not None: + base_key += "-tm-" + threat_model_digest + if extra_context_digest is not None: + base_key += "-xc-" + extra_context_digest + return base_key + +#: Parameterized on the *digests* of the documents that fed the prompt (not the +#: documents) so a recorded digest — e.g. from a run's cache tags — can rebuild the key. +#: Each input contributes a suffix only when present, so a run with neither document +#: keeps the bare ``bug_analysis`` key. +BUG_ANALYSIS_KEY = KeyFamily(ComponentGroup, _BugAnalysisCache, _bug_analysis_key) class _AgentResult(_BugAnalysisCache): final_history: list[AnyMessage] -def agent_round_key( - i: int -) -> CacheKey[_AgentResult, _AgentRoundWithHistory]: - return CacheKey[_AgentResult, _AgentRoundWithHistory](f"round-{i}") +def _agent_round_key(i: int) -> str: + return f"round-{i}" + +AGENT_ROUND_KEY = KeyFamily(_AgentResult, _AgentRoundWithHistory, _agent_round_key) AGENT_RESULT_KEY = CacheKey[_BugAnalysisCache, _AgentResult]("agent_bug_analysis") @@ -267,7 +270,7 @@ async def _run_bug_round( prev: list[_AgentRoundResult], system_prompt: str ) -> _AgentRoundWithHistory: - round_ctx = ctx.child(agent_round_key(round)) + round_ctx = ctx.child(AGENT_ROUND_KEY(round)) if (cached := await round_ctx.cache_get(_AgentRoundWithHistory)) is not None: return cached @@ -293,6 +296,8 @@ class ST(MessagesState, RoughDraftState): env.analysis_tools ).with_sys_prompt( system_prompt + ).with_monitor( + budget_monitor() ).compile_async() flow_input: BugAnalysisInput = BugAnalysisInput( @@ -342,6 +347,11 @@ async def _run_bug_analysis_inner[U: FeatureUnit]( }).render_to(load_jinja_template) for i in range(0, max_rounds): + # Under budget pressure a fresh round would be told to pack it in on + # its first monitor tick — don't bother launching it. Round 0 always + # runs (the loop's invariants require at least one round's history). + if i > 0 and budget_pressure(): + break next_result = await _run_bug_round( env, agent_component_analysis, i, initial_prompt_builder, prev_rounds, system_prompt ) @@ -368,6 +378,7 @@ async def run_property_inference[U: FeatureUnit]( component: U, extra_input : Sequence[AnyPropertyGenerationInput] = tuple(), threat_model: Document | None = None, + extra_context: Sequence[Document] = (), refinement: ConversationContextProvider | None = None, max_rounds: int = 3, *, @@ -383,9 +394,18 @@ async def run_property_inference[U: FeatureUnit]( tests, each describing what is and isn't a fit for *their* verification surface. It lives in the system prompt (rendered once) rather than the per-round initial prompt so it stays inside the cached prefix. + + ``extra_context`` is any number of documents the user supplied about the application. + Unlike ``threat_model`` they make no claim to be a list of *threats* — they are + presented as authoritative background, not a checklist. Rendered in the order given, + each labelled with its filename. """ - component_analysis = ctx.child(bug_analysis_key(threat_model, refinement is not None)) + component_analysis = ctx.child(BUG_ANALYSIS_KEY( + threat_model.to_digest() if threat_model is not None else None, + refinement is not None, + combine_digests([d.to_digest() for d in extra_context]), + )) if (cached := await component_analysis.cache_get(_BugAnalysisCache)) is not None: return cached.items @@ -401,14 +421,41 @@ def to_cache_level(s: bool) -> CacheLevel: actual_extra_input.append(CacheablePropertyGenerationInput( "certora:thread_model", "generic", "always", provide=lambda cache: [ - "In addition, a coworker has already written a 'threat model' for this application, which may include vulnerabilities/issues that" + "In addition, a coworker has already written a 'threat model' for this application, which may include vulnerabilities/issues that " "are common in this type of application. This threat model is written for the entire application (not just the component you are analyzing) " "so some of the issues/vulnerabilities/attacks may not be relevant to your analysis. Do *NOT* overfit to this threat model; carefully " "analyze what content of the provided threat model is worth considering vs out of scope. Further, this threat model is just a starting point, " - "you should ALSO look for threats *not* mentioned in this document.", + "you should ALSO look for threats *not* mentioned in this document.\n\n", threat_model.to_dict(cache_level=to_cache_level(cache)) ] )) + if extra_context: + def extra_context_blocks(cache: bool) -> list[RawMessageType]: + # Each document introduced by its filename, then its body. Only the final + # body carries the cache marker: a breakpoint is a prefix boundary, so + # marking an interior one spends one of the four without extending the + # cached prefix. + blocks: list[RawMessageType] = [] + for i, doc in enumerate(extra_context): + is_last = i == len(extra_context) - 1 + blocks.append(f"--- {doc.basename} ---\n\n") + blocks.append(doc.to_dict(cache_level=to_cache_level(cache and is_last))) + return blocks + + actual_extra_input.append(CacheablePropertyGenerationInput( + "certora:extra_context", "generic", "always", + provide=lambda cache: [ + f"In addition, the user requesting this analysis has provided the following extra context about " + f"the application ({len(extra_context)} document(s), each introduced by its filename below): " + "notes, assumptions, deployment details, or anything else they considered relevant. It was " + "written for the entire application (not just the component you are analyzing), so parts of it " + "may not bear on your analysis. Treat it as authoritative information about how the system is " + "intended to work and be deployed, and let it inform which behaviors count as violations. It " + "is *NOT* a list of the issues to look for and it is *NOT* exhaustive: keep looking for " + "problems it does not mention.\n\n", + *extra_context_blocks(cache), + ] + )) agent_attempt = await _run_bug_analysis_inner( diff --git a/composer/spec/solana/model.py b/composer/spec/solana/model.py index e9c460ae..1b930b68 100644 --- a/composer/spec/solana/model.py +++ b/composer/spec/solana/model.py @@ -280,7 +280,7 @@ def sibling_programs(self) -> list[SolanaProgram]: # -- FeatureUnit protocol ----------------------------------------------------------- @property - def display_name(self) -> str: + def display_name(self) -> ComponentName: return self.component.name @property diff --git a/composer/spec/solana/null_backend.py b/composer/spec/solana/null_backend.py index c76d8a0a..4cb73836 100644 --- a/composer/spec/solana/null_backend.py +++ b/composer/spec/solana/null_backend.py @@ -1,20 +1,21 @@ """A null Solana backend — records extracted properties without verifying them. -It satisfies the full ``PipelineBackend`` contract over the Solana ecosystem's +It implements the full ``PipelineBackend`` contract over the Solana ecosystem's ``(SolanaApplication, SolanaProgramInstance, SolanaComponentInstance)`` triple, but its ``formalize`` just echoes the extracted properties into a trivial result and its ``fetch_verdicts`` returns nothing. **Role:** a **test double** for the Solana front half (analysis + property extraction) without a real verifier — see ``tests/test_solana_gate.py``. Production Solana -verification is the Crucible fuzzer backend. +verification is the Crucible fuzzer backend — a Rust wheel hosted by +:mod:`composer.rustapp`. """ import enum import json from dataclasses import dataclass from pathlib import Path -from typing import override +from typing import override, Sequence, Any from pydantic import BaseModel, Field @@ -25,18 +26,19 @@ PipelineRun, PreparedSystem, SystemAnalysisSpec, + ToolBinder, ) from composer.spec.artifacts import ArtifactStore from composer.spec.context import WorkflowContext -from composer.spec.cvl_generation import SkippedProperty +from composer.authoring.state import SkippedProperty from composer.spec.solana.model import ( SolanaApplication, SolanaComponentInstance, SolanaProgramInstance, ) -from composer.spec.source.report.collect import ReportComponentInput, Verdict +from composer.spec.source.report.collect import Formalized, Verdict from composer.spec.source.report.schema import RuleName -from composer.spec.types import PropertyFormulation +from composer.spec.types import PropertyFormulation, PropertyTitle from composer.spec.util import ensure_dir SOLANA_NULL_GUIDANCE: str = """\ @@ -58,16 +60,16 @@ class NullResult(BaseModel): """A trivial formalization result: it just carries the properties back out.""" commentary: str = "" - property_rules: list[tuple[str, list[str]]] = Field(default_factory=list) + property_rules: list[tuple[PropertyTitle, list[RuleName]]] = Field(default_factory=list) skipped: list[SkippedProperty] = Field(default_factory=list) - def property_units(self) -> list[tuple[str, list[str]]]: + def property_checks(self) -> list[tuple[PropertyTitle, list[RuleName]]]: return [(t, list(u)) for t, u in self.property_rules] @property def artifact_text(self) -> str: return json.dumps( - {"commentary": self.commentary, "properties": self.property_units()}, indent=2 + {"commentary": self.commentary, "properties": self.property_checks()}, indent=2 ) @property @@ -92,7 +94,7 @@ class NullSolanaArtifactStore(ArtifactStore[NullArtifact, NullResult]): def __init__(self, project_root: str): super().__init__( project_root, - "property_units", + "property_checks", deliverable_dir="certora/solana_null", internal_dir=".certora_internal/solana_null", report_dir="certora/solana_null/reports", @@ -105,13 +107,10 @@ def _artifact_dir(self) -> Path: class NullSolanaFormalizer(Formalizer[NullResult, SolanaComponentInstance]): def __init__(self) -> None: - # The tag is provenance only — it picks the report's outcome labels, and this backend's - # results are all-UNKNOWN either way. So borrow ``"prover"``, an existing member of - # ``ReportBackend``: the real Solana verifier's own tag is added to that literal by the - # backend that introduces it, and until then a value outside the literal is not merely - # untyped but unusable — ``AutoProverReport`` is a pydantic model, so it would fail - # validation in ``build_report`` and lose the report phase to a swallowed exception. - super().__init__(NullResult, "prover") + # ``"none"``: this backend verifies nothing, and its report should say so rather than + # borrow a real verifier's vocabulary — every unit comes out UNKNOWN, which that tag's + # wording renders as "Unverified". + super().__init__(NullResult, "none") @override async def formalize( @@ -121,16 +120,19 @@ async def formalize( props: list[PropertyFormulation], ctx: WorkflowContext[NullResult], run: PipelineRun, + extra_tools: ToolBinder[SolanaComponentInstance] ) -> NullResult | GaveUp: return NullResult( commentary=f"Null formalization of instruction {feat.display_name} " f"({len(props)} properties recorded, unverified).", - property_rules=[(p.title, [p.title]) for p in props], + # The pseudo-check is named after the title itself: nothing runs, so the property's + # own words are the only name its report row could have. + property_rules=[(p.title, [RuleName(p.title)]) for p in props], ) @override async def fetch_verdicts( - self, inp: ReportComponentInput[NullResult] + self, formalized: Formalized[NullResult] ) -> dict[RuleName, Verdict]: return {} @@ -149,7 +151,7 @@ async def prepare_formalization( @dataclass class NullSolanaBackend: """``PipelineBackend[SolanaPhase, NullResult, None, NullArtifact, SolanaComponentInstance, - SolanaProgramInstance, SolanaApplication]`` (P, FormT, H, A, Unit, Main, App) — structural.""" + SolanaProgramInstance, SolanaApplication, None]`` (P, FormT, H, A, Unit, Main, App, Pre) — structural.""" artifact_store: NullSolanaArtifactStore backend_guidance = SOLANA_NULL_GUIDANCE @@ -163,8 +165,12 @@ class NullSolanaBackend: } ) + async def preflight(self, run: PipelineRun[SolanaPhase, None]) -> None: + """Nothing to prepare — this backend builds nothing and only records properties.""" + return None + async def prepare_system( - self, analyzed: SolanaApplication, run: PipelineRun[SolanaPhase, None] + self, analyzed: SolanaApplication, run: PipelineRun[SolanaPhase, None], preflight: None ) -> PreparedSystem[NullResult, SolanaComponentInstance, SolanaProgramInstance]: # Use the Solana ecosystem's locate_main so the backend and ecosystem agree on the # target program (imported lazily to avoid an import cycle with pipeline.ecosystem). diff --git a/composer/spec/soroban/__init__.py b/composer/spec/soroban/__init__.py new file mode 100644 index 00000000..a5bfcb00 --- /dev/null +++ b/composer/spec/soroban/__init__.py @@ -0,0 +1,4 @@ +"""Soroban/Stellar system model and unit wrappers. + +The `SOROBAN` ecosystem binding lives in `composer.pipeline.ecosystem`. +""" diff --git a/composer/spec/soroban/model.py b/composer/spec/soroban/model.py new file mode 100644 index 00000000..d2e9fb73 --- /dev/null +++ b/composer/spec/soroban/model.py @@ -0,0 +1,258 @@ +"""Soroban system model.""" + +from dataclasses import dataclass +from functools import cached_property +from typing import Literal + +from pydantic import BaseModel, Field + +from composer.spec.system_model import BaseApplication +from composer.spec.types import ComponentName, ContractName, RustIdentifier +from composer.spec.util import slugify_filename + +StorageDurability = Literal["instance", "persistent", "temporary"] + +StorageOperation = Literal["read", "write", "remove", "extend_ttl"] + +AuthKind = Literal["require_auth", "require_auth_for_args"] + + +class StorageEntry(BaseModel): + key: str = Field( + description="Source key, such as a DataKey variant (`Balance(Address)`) or a Symbol." + ) + durability: StorageDurability = Field( + description="Storage kind used at the call site: instance, persistent, or temporary." + ) + value_type: str = Field(description="The stored type.") + description: str = Field(description="What this entry stores and who may change it.") + + +class AuthRequirement(BaseModel): + address: str = Field( + description="Address being authorized, as an argument name or stored address." + ) + kind: AuthKind = Field( + description="Auth call used: require_auth or require_auth_for_args." + ) + description: str = Field(description="What this auth check protects.") + + +class StorageAccessSite(BaseModel): + key: str = Field(description="Storage key accessed.") + durability: StorageDurability = Field( + description="Storage kind used for this access." + ) + access: StorageOperation = Field( + description="Operation: read, write, remove, or extend_ttl." + ) + + +class ContractCall(BaseModel): + target_contract: str = Field( + description="Called contract, or how its address is found." + ) + description: str = Field( + description="What is called and how errors are handled." + ) + + +class SorobanFunction(BaseModel): + name: str = Field(description="The function's snake_case name.") + description: str = Field(description="What the function does.") + args: list[str] = Field( + default_factory=list, description="Arguments, excluding `env`." + ) + returns: str | None = Field(default=None, description="The return type, or null.") + auth: list[AuthRequirement] = Field( + default_factory=list, + description="Auth checks performed by the function. Empty means no auth.", + ) + storage: list[StorageAccessSite] = Field( + default_factory=list, description="Storage accessed by this function." + ) + calls: list[ContractCall] = Field( + default_factory=list, description="Cross-contract calls made by this function." + ) + events: list[str] = Field(default_factory=list, description="Events emitted.") + errors: list[str] = Field( + default_factory=list, + description="Failure cases, including contract errors, panics, unwraps, overflow, or conversion failures.", + ) + requirements: list[str] = Field( + description="Required function behavior." + ) + + def to_signature(self) -> str: + """This function rendered as a Rust-like signature, e.g. + ``transfer(from: Address, to: Address, amount: i128) -> Result<(), Error>``. + + Note: ``env`` is already excluded from ``args`` by the analysis prompt, + so this looks like what the caller sees.""" + returns = f" -> {self.returns}" if self.returns else "" + return f"{self.name}({', '.join(self.args)}){returns}" + + +class InterComponentInteraction(BaseModel): + contract: ContractName = Field( + description="Name of the contract used." + ) + # ``_soroban_validate`` rejects a name that does not resolve. + component: ComponentName = Field( + description="The component of that contract this interaction is with. Must match the " + "`name` of a component declared on it." + ) + description: str = Field(description="How this component is used.") + + +class AuthorityInteraction(BaseModel): + authority: str = Field( + description="Name of the external actor used." + ) + description: str = Field(description="How this actor is used.") + + +type ComponentInteraction = InterComponentInteraction | AuthorityInteraction + + +class ContractComponent(BaseModel): + name: ComponentName = Field(description="Short component name.") + description: str = Field( + description="What this component does." + ) + functions: list[str] = Field( + description="Function names in this component." + ) + storage_keys: list[str] = Field( + description="Storage keys maintained by this component." + ) + interactions: list[ComponentInteraction] = Field( + description="Links to other components or external actors." + ) + requirements: list[str] = Field( + description="Required component behavior." + ) + + +class SorobanContract(BaseModel): + name: ContractName = Field( + description="Short contract name." + ) + contract_identifier: RustIdentifier = Field( + pattern=r"^[a-zA-Z_][a-zA-Z0-9_]*$", + description="Rust identifier for the contract type, crate, or module.", + ) + contract_id: str | None = Field( + default=None, + description="Pinned deployed contract address, if any.", + ) + description: str = Field(description="The contract's role in the system.") + functions: list[SorobanFunction] = Field(description="The contract's entry points.") + storage_entries: list[StorageEntry] = Field( + default_factory=list, description="The storage entries this contract owns." + ) + components: list[ContractComponent] = Field( + description="Feature groups in this contract. Every function must appear in at least one." + ) + + @cached_property + def functions_by_name(self) -> dict[str, SorobanFunction]: + return {f.name: f for f in self.functions} + + @cached_property + def storage_by_key(self) -> dict[str, StorageEntry]: + return {e.key: e for e in self.storage_entries} + + +class SorobanAuthority(BaseModel): + name: str = Field(description="Short external actor name.") + description: str = Field(description="What this actor does.") + assumptions: list[str] = Field( + default_factory=list, + description="Trust assumptions. For SACs, include issuer mint, clawback, and set_authorized powers.", + ) + + +type SorobanComponent = SorobanContract | SorobanAuthority + + +class SorobanApplication(BaseApplication[SorobanComponent]): + @cached_property + def contracts(self) -> list[SorobanContract]: + return [c for c in self.components if isinstance(c, SorobanContract)] + + @cached_property + def authorities(self) -> list[SorobanAuthority]: + return [c for c in self.components if isinstance(c, SorobanAuthority)] + + +@dataclass +class SorobanContractInstance: + ind: int + app: SorobanApplication + + @property + def contract(self) -> SorobanContract: + return self.app.contracts[self.ind] + + +@dataclass +class SorobanComponentInstance: + ind: int + _contract: SorobanContractInstance + + @property + def app(self) -> SorobanApplication: + return self._contract.app + + @property + def contract(self) -> SorobanContract: + return self._contract.contract + + @property + def component(self) -> ContractComponent: + return self.contract.components[self.ind] + + @property + def functions(self) -> list[SorobanFunction]: + by_name = self.contract.functions_by_name + return [by_name[n] for n in self.component.functions if n in by_name] + + @property + def storage_entries(self) -> list[StorageEntry]: + by_key = self.contract.storage_by_key + return [by_key[k] for k in self.component.storage_keys if k in by_key] + + @property + def sibling_components(self) -> list[ContractComponent]: + return [c for i, c in enumerate(self.contract.components) if i != self.ind] + + @property + def sibling_contracts(self) -> list[SorobanContract]: + return [c for i, c in enumerate(self.app.contracts) if i != self._contract.ind] + + @property + def display_name(self) -> ComponentName: + return self.component.name + + @property + def slug(self) -> str: + return slugify_filename(self.component.name) + + @property + def unit_index(self) -> int: + return self.ind + + def cache_material(self) -> str: + return "|".join([self.app.model_dump_json(), str(self.ind), str(self._contract.ind)]) + + def context_tag(self) -> dict[str, object]: + return {"component": self.component.model_dump()} + + def feature_json(self) -> dict[str, object]: + return { + **self.component.model_dump(mode="json"), + "slug": self.slug, + "functions": [f.model_dump(mode="json") for f in self.functions], + "storage_entries": [e.model_dump(mode="json") for e in self.storage_entries], + } diff --git a/composer/spec/source/author.py b/composer/spec/source/author.py index 351843ef..7fb6a900 100644 --- a/composer/spec/source/author.py +++ b/composer/spec/source/author.py @@ -1,6 +1,9 @@ -from typing import NotRequired, Sequence, override, Literal, Annotated +from typing import AsyncIterator, NotRequired, override, Literal, Annotated, Sequence, Protocol, Callable + from typing_extensions import TypedDict +from contextlib import asynccontextmanager import json +import pathlib from dataclasses import dataclass @@ -11,23 +14,31 @@ WithAsyncImplementation, WithImplementation, WithInjectedId, WithInjectedState, WithAsyncDependencies ) +from graphcore.graph import tool_state_update, RawPromptInput, CacheMarker, SummaryConfig from graphcore.tools.vfs import VFSAccessor, VFSState -from graphcore.graph import tool_state_update -from graphcore.summary import SummaryConfig +from composer.authoring.judge import PropertyFeedbackProtocol +from composer.authoring.state import SkippedProperty, check_completion +from composer.authoring.tools import give_up_tool from composer.spec.cvl_generation import ( static_tools, property_tools, skip_tools, CVLGenerationExtra, FEEDBACK_VALIDATION_KEY, - check_completion, validate_property_rules, CVL_JUDGE_KEY, run_cvl_generator, - GeneratedCVL, PropertyRuleMapping, AppliedEdit, FeedbackToolBase, SkippedProperty, - PropertyFeedbackProtocol, + validate_property_rules, CVL_JUDGE_KEY, run_cvl_generator, + GeneratedCVL, PropertyRuleMapping, AppliedEdit, FeedbackToolBase, ) +from composer.prover.core import run_prover, CexHandler, ProverCallbacks, ProverReport +from composer.spec.source.autosetup import read_summarization_candidates +from summarization_detector.schema import HostileCandidate from composer.spec.source.live_explorer import VersionedHistory, LiveEditTools, WIPE_HISTORY +from composer.spec.source.prover import setup_prover_config_in from composer.spec.context import WorkflowContext, CVLGeneration, CacheKey, SourceCode -from composer.spec.types import PropertyFormulation -from composer.pipeline.core import GaveUp +from composer.spec.types import PropertyFormulation, PropertyTitle +from composer.pipeline.core import GaveUp, ToolBinder, InjectingToolExtension, Curtailed +from composer.pipeline.plugin_api import ProvidedTools +from composer.spec.source.plugin import CertoraProverTools, CVLAuthorState from composer.spec.system_model import ContractComponentInstance, SolidityIdentifier, component_context from composer.spec.source.prover import ( OVERLAY_OWNED_KEYS, ProverStateExtra, DELETE_SKIP, VALIDATION_KEY as PROVER_VALIDATION_KEY, + materializing_project, ) from langgraph.graph import MessagesState from pathlib import Path @@ -38,6 +49,8 @@ from composer.pipeline.ptypes import GaveUp +from composer.prover.core import ProverOptions + from langgraph.types import Command from graphcore.graph import Builder from composer.spec.feedback import ( @@ -45,21 +58,32 @@ SourceSnapshot, ContextualFeedbackToolImpl, ) from composer.ui.tool_display import tool_display +from composer.diagnostics.budget import ( + BudgetExceeded, budget_monitor, constraint_sort_to_noun, raise_budget_exceeded +) +from .monitor import monitor from composer.spec.source.conf_maps import ( CompilerSettings, MapViolations, extend_compiler_maps, map_violation_message, ) -from composer.spec.source.munge.edit_store import EditStore +from composer.spec.source.munge.edit_store import EditStore, PluginEditor from composer.spec.source.munge.munge_agent import editor_tool from composer.spec.source.munge.tool_names import ( COMMIT_EDIT, CONFIG_EDIT, EDIT_HISTORY_LOG, REVERT_TO_EDIT, ) from composer.spec.source.munge.vfs_diff import summarize_changes -from graphcore.graph import FlowInput +from graphcore.graph import FlowInput, MonitorReturn + +from composer.io.task_host import TaskHost +from composer.io.task_tools import RETRIEVE_TASK, TASK_LIST, RetrieveTask, TaskListTool class SourceAuthorExtra(TypedDict): failed: bool | None + #: Stamped True by the budget monitor's state transformer when the wrap-up alert fires (the + #: same update that lifts the validation gates), so the published result is known to be a + #: budget-curtailed partial rather than a validated delivery. + budget_curtailed: bool # ``vfs`` comes from ProverStateExtra (NotRequired, no merge op — replaced # wholesale by commit_edit / revert_to_edit); the generation input always @@ -73,7 +97,7 @@ class SourceCVLGenerationInput(SourceCVLGenerationExtra, FlowInput): class SourceCVLGenerationState(SourceCVLGenerationExtra, MessagesState): result: NotRequired[str] -type BatchGeneratedCVLResult = GeneratedCVL | GaveUp +type BatchGeneratedCVLResult = GeneratedCVL | Curtailed[GeneratedCVL] | GaveUp @tool_display(lambda p: f"Expecting rule `{p['rule_name']}` to fail", None) class ExpectRuleFailure(WithAsyncImplementation[Command], WithInjectedId): @@ -117,7 +141,7 @@ async def run(self) -> Command: result=None, ) class PublishResultTool( - WithAsyncDependencies[Command | str, list[str]], + WithAsyncDependencies[Command | str, list[PropertyTitle]], WithInjectedState[SourceCVLGenerationState], WithInjectedId, ): @@ -148,27 +172,13 @@ async def run(self) -> Command | str: ) -@tool_display( - label=lambda p: f"Giving up on CVL generation: {p['reason']}", - result=None, -) -class GiveUpTool(WithImplementation[Command], WithInjectedId): - """ +_GIVE_UP_DESCRIPTION = """ Call this tool to give up on the CVL generation for this task. This should only ever be called as a LAST RESORT when you have exhausted all other mechanisms to complete your task. """ - reason: str = Field(description="The reason for giving up on your task") - @override - def run(self) -> Command: - return tool_state_update( - self.tool_call_id, - "Accepted", - failed=True, - result=self.reason, - ) class ResourceView(TypedDict): """A CVLResource prepared for the prompt: ``import_path`` is the CVL import @@ -184,6 +194,7 @@ class PropertyGenParams(TypedDict): resources: list[ResourceView] properties: list[PropertyFormulation] contract_name: str + hostile_candidates: list[HostileCandidate] # detector's prover-hostile summarization targets (may be empty) class PropertyGenerationConfig(SummaryConfig[SourceCVLGenerationState]): def __init__(self, source_editing: bool = False): @@ -532,6 +543,18 @@ class SourceEditing: store: EditStore +@dataclass(frozen=True) +class EditingTools: + """Source editing and plugin tool contribution, fused: a contributed tool's + staged :class:`CVLAuthorState` proposes edits into the editing kit's store + and materializes against its working copy, so a binder without an editing + kit is unusable. Fusing them makes "tools provided ⟺ editing enabled" a + fact of the type rather than an assert (the structural-invariant phase + passes None: neither).""" + editing: SourceEditing + tool_provider: ToolBinder[ContractComponentInstance] + + class _LastAttemptEdits(BaseModel): """Sibling of cvl_generation's last-attempt draft cache: the applied-edit history at snapshot time. The working copy itself is deliberately not @@ -608,25 +631,114 @@ def _version_history(self) -> Sequence[str]: _PropertyGenTemplate = TypedTemplate[PropertyGenParams]("property_generation_prompt.j2") +class PropertyGenSystemParams(TypedDict): + source_editing: bool + +_PropertyGenSysTemplate = TypedTemplate[PropertyGenSystemParams]("property_generation_system_prompt.j2") + +#: The prover's tool extension: contributions come from plugins deriving +#: ``CertoraProverTools``, dispatched via their ``certora_prover_tools`` hook. +_PROVER_TOOLS = InjectingToolExtension( + provider=CertoraProverTools, project=lambda p: p.certora_prover_tools +) + +@dataclass +class ProverTool: + lg_tool: BaseTool + options: ProverOptions + +@dataclass +class WrappedProverRunner: + config: dict + prover_options: ProverOptions + main_contract: str + + async def run( + self, + *, + curr_spec: str, + working_dir: str, + cex_handler: CexHandler, + callbacks: ProverCallbacks, + tool_call_id: str, + rules: list[str] | None = None, + **config, + ) -> ProverReport | str: + # The spec/conf staging only has to outlive the run itself, so one call + # stages, runs, and cleans up (the CVLAuthorState.prover_runner contract). + with setup_prover_config_in( + working_dir=working_dir, + spec_stem="adhoc_run", + main_contract=self.main_contract, + spec_contents=curr_spec, + config=self.config, + rule=rules, + **config + ) as (conf_path, _): + return await run_prover( + pathlib.Path(working_dir), + [conf_path], + tool_call_id, + self.prover_options, + callbacks, cex_handler + ) + + +_BUDGET_WRAPUP_MESSAGE = """ + +You have almost exceeded the {resource} budget for this task. Wrap up IMMEDIATELY; +a partial spec is better than going over budget. Concretely: + +- The feedback and prover validation requirements on publishing have been lifted. You no + longer need approval from the feedback judge — ignore any pending or future feedback, + including a judge response saying it was terminated. +- Do NOT start new prover runs or research. +- Delete any rules/invariants that do not currently work. +- Skip (`record_skip`) every property you have not gotten to work, citing budget exhaustion. +- Then publish what remains via the `result` tool. + +""" + + +def _author_monitor() -> Callable[[SourceCVLGenerationState], MonitorReturn]: + """The author's monitor: budget wrap-up takes precedence; otherwise the + usual reminders-channel drain. On the (single) turn the budget warning + fires any pending reminders are dropped — moot, since the warning tells + the agent to ignore prover/feedback outcomes anyway.""" + b_monitor = budget_monitor( + warning_message=lambda _s, c: _BUDGET_WRAPUP_MESSAGE.format(resource=constraint_sort_to_noun(c)), + state_transformer=lambda _s, _c: {"required_validations": [], "budget_curtailed": True}, + on_overbudget=raise_budget_exceeded, + ) + + def combined(curr_state: SourceCVLGenerationState) -> MonitorReturn: + msgs, upd = b_monitor(curr_state) + if msgs is None and upd is None: + return monitor(curr_state) + return msgs, upd + + return combined + async def batch_cvl_generation( ctx: WorkflowContext[CVLGeneration], init_config: dict, props: list[PropertyFormulation], component: ContractComponentInstance | None, resources: list[CVLResource], - prover_tool: BaseTool, + prover_tool: ProverTool, env: ServiceHost, description: str, source: SourceCode, spec_dir: Path, spec_stem: str, - editing: SourceEditing | None, + editing_tools: EditingTools | None, ) -> BatchGeneratedCVLResult: # *spec_dir* (project-root-relative) is where the caller will persist the spec # authored here. The prover resolves the spec's CVL imports relative to its own # directory, so resource imports are expressed relative to *spec_dir*. # *spec_stem* is the basename it is persisted under; the prover materializes its # transient spec/conf under the same stem so on-disk names match the dump. + editing = editing_tools.editing if editing_tools is not None else None resource_views: list[ResourceView] = [ { "description": r.description, @@ -640,9 +752,77 @@ async def batch_cvl_generation( "context": component, "properties": props, "contract_name": source.contract_name, - "sort": "existing" + "sort": "existing", + "hostile_candidates": read_summarization_candidates(Path(source.project_root)), }) + sys_prompt : list[RawPromptInput | type[CacheMarker]] = [ + _PropertyGenSysTemplate.bind({"source_editing": editing is not None}).render_to + ] + + added_tools : list[BaseTool] = [] + if editing_tools is not None: + task_host = TaskHost() + kit = editing_tools.editing + # The same run-root strategy verify_spec uses (see ProjectDirectory): an + # empty working copy is read in-situ, a non-empty one against a temporary + # materialization whose lifetime is the contributed tool's invocation. + project_directory = materializing_project(source.project_root, kit.live.mat) + + @asynccontextmanager + async def yield_state( + plugin_id: str, + st: SourceCVLGenerationState + ) -> AsyncIterator[CVLAuthorState]: + class _PluginStore: + async def propose( + self, vfs: dict[str, str], *, executive_summary: str, why_sound: str + ) -> str: + # Snapshot completion (the EditProposer contract): the + # proposer's overlay is relative to the working copy this + # read staged, but ApplyEditTool snapshots are wholesale — + # so fold the author's own overlay back in, or applying + # the proposal would silently revert prior edits. + return await kit.store.commit( + {**(st.get("vfs") or {}), **vfs}, + executive_summary=executive_summary, + why_sound=why_sound, + attribution=PluginEditor(plugin_id) + ) + async with project_directory(st.get("vfs") or {}) as run_root: + yield CVLAuthorState( + working_dir=pathlib.Path(run_root), + curr_spec=st["curr_spec"], + prover_runner=WrappedProverRunner( + st["config"], + prover_tool.options, + source.contract_name + ).run, + host=task_host, + edit_store=_PluginStore() + ) + + tools = await editing_tools.tool_provider( + _PROVER_TOOLS, yield_state, SourceCVLGenerationState + ) + if tools: + # The retrieval surface for whatever the contributed tools launch; + # dead prompt weight when no plugin contributed, so gated on a + # non-empty contribution. + added_tools.extend([ + TaskListTool.bind(task_host).as_tool(TASK_LIST), + RetrieveTask.bind(task_host).as_tool(RETRIEVE_TASK), + ]) + else: + tools = [] + + for inj in tools: + added_tools.extend(inj.tools) + if isinstance(inj.system_prompt_injection, list): + sys_prompt.extend(inj.system_prompt_injection) + else: + sys_prompt.append(inj.system_prompt_injection) + titles = [p.title for p in props] judge_ctx = ctx.child(CVL_JUDGE_KEY) judge_prompt = FeedbackTemplate.bind({ @@ -685,22 +865,26 @@ async def batch_cvl_generation( ).with_tools( feedback_suite ).with_tools( - [prover_tool, + [prover_tool.lg_tool, ExpectRulePassage.as_tool("expect_rule_passage"), ExpectRuleFailure.as_tool("expect_rule_failure"), - GiveUpTool.as_tool("give_up"), + give_up_tool(name="give_up", description=_GIVE_UP_DESCRIPTION, label="CVL generation"), PublishResultTool.bind(titles).as_tool("result"), ctx.get_memory_tool()] ).with_state( SourceCVLGenerationState + ).with_monitor( + _author_monitor() ).with_output_key( "result" ).with_input( SourceCVLGenerationInput - ).with_sys_prompt_template( - "property_generation_system_prompt.j2", source_editing=editing is not None + ).with_sys_prompt( + [*sys_prompt] ).with_initial_prompt( with_cvl_context(bound_template.render_to) + ).with_tools( + added_tools ).with_summary_config( PropertyGenerationConfig(source_editing=editing is not None) ).compile_async() @@ -745,10 +929,16 @@ async def batch_cvl_generation( property_rules=[], validations={}, failed=None, + budget_curtailed=False, + prover_history=[], + reminders_channel=[], vfs=restored_vfs, - version_history=restored_history + version_history=restored_history, + spec_stem=spec_stem ) ) + except BudgetExceeded as e: + return Curtailed(None, detail=str(e)) finally: if editing is not None: last_state = ( @@ -765,6 +955,10 @@ async def batch_cvl_generation( assert "result" in res_state assert res_state["failed"] is not None if res_state["failed"]: + if res_state["budget_curtailed"]: + # A give-up issued after the wrap-up order isn't a considered "this batch is + # unformalizable" judgment — it's the budget talking. Keep the agent's account. + return Curtailed(None, detail=res_state["result"]) return GaveUp(reason=res_state["result"]) d = res_state["curr_spec"] assert d is not None @@ -780,8 +974,9 @@ async def batch_cvl_generation( )) # Persist the base prover config and last run link from the final state so a later cache # hit (which skips the prover) can still reconstruct certora/confs and retain the link. + assert "vfs" in res_state - return GeneratedCVL( + generated = GeneratedCVL( commentary=res_state["result"], cvl=d, skipped=res_state["skipped"], @@ -791,4 +986,8 @@ async def batch_cvl_generation( vfs=res_state["vfs"], applied_edits=applied_edits, ) + if res_state["budget_curtailed"]: + # Published under lifted gates: hand it back as an explicitly unreliable partial. + return Curtailed(generated) + return generated diff --git a/composer/spec/source/autoprove_common.py b/composer/spec/source/autoprove_common.py index 953e340e..c37e8ca8 100644 --- a/composer/spec/source/autoprove_common.py +++ b/composer/spec/source/autoprove_common.py @@ -10,7 +10,7 @@ from composer.diagnostics.timing import RunSummary from composer.input.types import DEFAULT_RECURSION_LIMIT, ExtendedModelOptions, RAGDBOptions -from composer.input.parsing import add_protocol_args +from composer.input.parsing import add_extra_context_args, add_protocol_args from composer.rag.db import PostgreSQLRAGDatabase from composer.pipeline.core import CorePipelineResult @@ -18,6 +18,7 @@ SourceFields ) from composer.pipeline.cli import cli_pipeline, user_ns +from composer.pipeline.ptypes import DEFAULT_MAX_CPU_TASKS from composer.pipeline.ecosystem import EVM from composer.spec.source.pipeline import ProverBackend, GeneratedCVL from composer.spec.source.cex_capture import CexAnalysisStore @@ -48,13 +49,17 @@ class AutoProveArgs(ExtendedModelOptions, RAGDBOptions, Protocol): main_contract: str system_doc: str | None max_concurrent: int + max_cpu_tasks: int cache_ns: str | None memory_ns: str | None cloud: bool interactive: bool threat_model: str + extra_context: list[str] | None recursion_limit: int max_bug_rounds: int + budget: str | None + time_budget: float | None # --------------------------------------------------------------------------- @@ -75,12 +80,16 @@ async def _entry_point(summary: RunSummary) -> AsyncIterator[Executor]: parser.add_argument("main_contract", help="Main contract as path:ContractName") parser.add_argument("system_doc", nargs="?", default=None, help="Path to the design document (text or PDF). Optional — auto-discovered from the project when omitted.") parser.add_argument("--max-concurrent", type=int, default=4, help="Max concurrent agents (default: 4)") + parser.add_argument("--max-cpu-tasks", type=int, default=DEFAULT_MAX_CPU_TASKS, help=f"Max concurrent CPU-bound tasks — toolchain builds and the like (default: {DEFAULT_MAX_CPU_TASKS})") parser.add_argument("--cache-ns", default=None, help="Cache namespace (enables cross-run caching)") parser.add_argument("--memory-ns", default=None, help="Memory namespace (default: thread id)") parser.add_argument("--cloud", action="store_true", help="Run prover jobs in the cloud") parser.add_argument("--interactive", action="store_true", help="Interactively refine the security properties after extraction") - parser.add_argument("--threat-model", type=str, default=None, help="Path to a 'thread' model (text or pdf) with which to seed the property extraction process") + parser.add_argument("--threat-model", type=str, default=None, help="Path to a 'threat' model (text or pdf) with which to seed the property extraction process") + add_extra_context_args(parser) parser.add_argument("--max-bug-rounds", type=int, default=3, help="Maximum number of bug-extraction rounds run per component during property analysis (default: 3)") + parser.add_argument("--budget", default=None, help="Path to a run-budget file (JSON or YAML): {total: USD, caps: {phase: USD, ...}}. Omit to run unbudgeted.") + parser.add_argument("--time-budget", default=None, type=float, help="Total wall time to run the entire execution. Omit to run without in process limit") args = cast(AutoProveArgs, parser.parse_args()) async with autoprove_executor(args, summary) as runner: @@ -128,7 +137,6 @@ async def callback( thread_id=thread_id, task_handler=handler, at_exit=exit_logger, - workflow="autoprove" ) as (staged, cont), PostgreSQLRAGDatabase.rag_context(staged.embed_model, args.rag_db) as rag_db @@ -145,6 +153,7 @@ async def callback( source_question_ns=source_data_ns, recursion_limit=args.recursion_limit, cvl_index_config=agent_index_config_from_env(DEFAULT_CVL_AGENT_INDEX_NS), + ecosystem=EVM, ) # Source-editing kit: the edit snapshot store, the live (vfs-aware) # tool suite with its versioned explorer, and the migration oracle @@ -167,6 +176,7 @@ async def callback( source_key=staged.root_key, oracle=mk_oracle(edit_store, staged.source), recursion_limit=args.recursion_limit, + ecosystem=EVM, ), store=edit_store, ) diff --git a/composer/spec/source/autosetup.py b/composer/spec/source/autosetup.py index 338c064b..88aa44d2 100644 --- a/composer/spec/source/autosetup.py +++ b/composer/spec/source/autosetup.py @@ -24,7 +24,10 @@ from certora_autosetup.utils.paths import ( resolve_autosetup_llm_usage_file, resolve_autosetup_prover_usage_file, + resolve_autosetup_summarization_candidates_file, ) +# The detector owns the schema of the file it writes; import its typed view rather than re-declaring it. +from summarization_detector.schema import HostileCandidate _logger = logging.getLogger(__name__) @@ -139,6 +142,7 @@ def log_complete(self, returncode: int): "--composer-setup", f.name, "--no-strip-contracts", "--skip-harnessing", + "--skip-test-run", "--run-source", "AUTO_PROVER", "--main-contract", main_contract_path, @@ -275,3 +279,21 @@ def read_autosetup_prover_usage(project_root: Path) -> int | None: except (OSError, ValueError, KeyError, TypeError) as e: _logger.warning(f"Could not read AutoSetup prover usage from {usage_file}: {e}") return None + + +def read_summarization_candidates(project_root: Path) -> list[HostileCandidate]: + """The summarization detector's ranked candidates from the AutoSetup run — the prover-hostile + functions worth summarizing (function, category, reaching methods, why, and, for a curated match, a + suggested summary), rendered into the CVL-generation and invariants agent prompts. + + AutoSetup writes ``summarization_candidates.json`` after its test run; returns ``[]`` on any failure + (file absent — the run predates the detector, a full cache hit, or the test run dumped no surviving + graph — or malformed JSON): a missing hint must never break the phase.""" + candidates_file = resolve_autosetup_summarization_candidates_file(project_root) + if candidates_file is None: + return [] + try: + return json.loads(candidates_file.read_text()).get("candidates", []) + except (OSError, ValueError, TypeError) as e: + _logger.warning(f"Could not read AutoSetup summarization candidates from {candidates_file}: {e}") + return [] diff --git a/composer/spec/source/harness.py b/composer/spec/source/harness.py index 7331042d..7486d6e2 100644 --- a/composer/spec/source/harness.py +++ b/composer/spec/source/harness.py @@ -18,27 +18,37 @@ compilation, and returns a ``Configuration``. """ -from typing import NotRequired, TypedDict +from typing import AsyncIterator, NotRequired, TypedDict, override +from contextlib import ExitStack, asynccontextmanager +from logging import getLogger from pathlib import Path -import subprocess +import asyncio +import shutil -from pydantic import Field, BaseModel +from pydantic import Field, BaseModel, ValidationError from langgraph.graph import MessagesState from composer.prover.core import ProverOptions +from composer.spec.natspec.async_result import AsyncResultTool from graphcore.graph import FlowInput -from graphcore.tools.vfs import VFSState, VFSToolConfig, vfs_tools -from graphcore.tools.results import result_tool_generator +from graphcore.tools.schemas import WithInjectedState +from graphcore.tools.vfs import VFSAccessor, VFSState, VFSToolConfig, vfs_tools from composer.diagnostics.timing import get_run_summary from composer.spec.graph_builder import run_to_completion, bind_standard from composer.spec.source.autosetup import run_autosetup, read_autosetup_usage, read_autosetup_prover_usage, SetupFailure, SetupSuccess from composer.spec.service_host import ServiceHost from composer.spec.context import WorkflowContext, SourceCode, CacheKey +from composer.spec.key_family import KeyFamily from composer.spec.util import string_hash from composer.spec.gen_types import TypedTemplate, certora_relative_to_project, under_project -from composer.spec.system_model import SolidityIdentifier, SourceApplication, SourceExternalActor, SourceExplicitContract +from composer.spec.system_model import ( + HarnessDefinition, HarnessedApplication, HarnessedExplicitContract, + SolidityIdentifier, SourceApplication, SourceExternalActor, SourceExplicitContract, +) + +_logger = getLogger(__name__) def system_setup_key(s: SourceApplication) -> CacheKey["ContractSetup", "SystemDescriptionHarnessed"]: return CacheKey["ContractSetup", "SystemDescriptionHarnessed"]( @@ -71,7 +81,22 @@ class ClosureContract(ClosureContractBase): """ A contract in the transitive closure. """ - num_instances : int | None = Field(description="The number of instances of this contract needed to model a non-trivial state (None if N/A)") + num_instances : int | None = Field(description="The number of instances (> 1) of this contract needed to model a non-trivial state; None if a single instance is sufficient") + +class HarnessingDetermination(BaseModel): + """ + A determination that a multiple harness instances are required for this contract + """ + n : int = Field(description="The number of harnesses needed for a non-trivial state; must be > 1", gt=1) + +class TaggedClosureContract(ClosureContractBase): + __doc__ = ClosureContract.__doc__ + + harness_determination : HarnessingDetermination | None = Field(description="The harnessing determination for this contract") + + @property + def num_instances(self) -> int | None: + return None if self.harness_determination is None else self.harness_determination.n class HarnessDef(BaseModel): harness_of: SolidityIdentifier @@ -99,7 +124,7 @@ class SystemDescriptionBase[T: ClosureContractBase](BaseModel): external_interfaces: list[ExternalInterface] = Field(description="A list of the external contract actors interacted with by the closure") -class AgentSystemDescription(SystemDescriptionBase[ClosureContract]): +class AgentSystemDescription(SystemDescriptionBase[TaggedClosureContract]): """ The result of your analysis """ @@ -128,6 +153,45 @@ class ContractSetup(BaseModel): system_description: SystemDescriptionHarnessed config: SetupSuccess + +def lift_harnessed( + s: SourceApplication, sys_desc: SystemDescriptionHarnessed, +) -> HarnessedApplication: + """Re-key harness definitions by harnessed contract and fold them into a + ``HarnessedApplication`` — each ``SourceExplicitContract`` becomes a + ``HarnessedExplicitContract`` carrying the harnesses generated for it. + + Declared beside the models because every consumer of the harnessed view + must build it identically — the prover pipeline, and any offline walker + re-deriving component cache keys (``composer.meta.run``): the harnessed + app is part of the unit's ``cache_material``.""" + contract_to_harness: dict[SolidityIdentifier, list[HarnessDefinition]] = {} + for c in sys_desc.transitive_closure: + if not c.harness_definition: + continue + contract_to_harness.setdefault(c.harness_definition.harness_of, []).append( + HarnessDefinition(name=c.solidity_identifier, path=c.path) + ) + + comp: list[SourceExternalActor | HarnessedExplicitContract] = [] + for c in s.components: + if not isinstance(c, SourceExplicitContract): + comp.append(c) + continue + comp.append(HarnessedExplicitContract( + sort=c.sort, + name=c.name, + solidity_identifier=c.solidity_identifier, + components=c.components, + description=c.description, + path=c.path, + harnesses=contract_to_harness.get(c.solidity_identifier, []), + )) + return HarnessedApplication( + application_type=s.application_type, description=s.description, components=comp, + ) + + HarnessAnalysis = TypedTemplate[HarnessAnalysisParams]("state_analysis.j2") HARNESS_ANALYSIS_KEY = CacheKey[SystemDescriptionHarnessed, AgentSystemDescription]("harness-analysis") @@ -199,6 +263,92 @@ def result_validator( await child.cache_put(res["result"]) return res["result"] +_HARNESS_CHECK_TIMEOUT_S = 900 + + +class ForgeDiagnostic(BaseModel): + """One diagnostic of a ``forge build`` report.""" + severity: str + message: str = "" + formatted_message: str = Field(default="", alias="formattedMessage") + + @property + def rendered(self) -> str: + """The diagnostic as solc rendered it: the source excerpt and the notes + that go with it, falling back to the bare message if forge gave none.""" + return self.formatted_message or self.message + + +class ForgeReport(BaseModel): + """A ``forge build --json`` report. + + ``errors`` holds every diagnostic, warnings included; ``severity`` is what + separates them. The exit code carries no information — forge exits 0 + whether or not the sources compiled. + """ + errors: list[ForgeDiagnostic] = Field(default_factory=list) + + @property + def compile_errors(self) -> list[str]: + return [d.rendered for d in self.errors if d.severity == "error"] + + +@asynccontextmanager +async def _materialized[S](accessor: VFSAccessor[S], state: S) -> AsyncIterator[str]: + """The agent's filesystem view as a real directory, for the block's + duration. The copy and its teardown run in a worker thread: materializing a + whole project is blocking IO that would otherwise stall the event loop.""" + stack = ExitStack() + project_dir = await asyncio.to_thread(stack.enter_context, accessor.materialize(state)) + try: + yield project_dir + finally: + await asyncio.to_thread(stack.close) + + +async def _compile_check(project_dir: str, harness_paths: list[str]) -> str | None: + """Type-check the named harnesses within a materialized copy of the project. + + Returns the compiler's error output, or None when they compile — or when + the project offers nothing to check them with (not a foundry project, no + forge binary, or a build that never produced a report), in which case the + candidates are accepted unchecked. + """ + root = Path(project_dir) + forge = shutil.which("forge") + if forge is None or not (root / "foundry.toml").is_file(): + _logger.info("Harness compile check skipped: needs forge and a foundry project") + return None + proc = await asyncio.create_subprocess_exec( + forge, "build", "--json", *sorted(harness_paths), + cwd=str(root), + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + ) + try: + stdout_b, stderr_b = await asyncio.wait_for( + proc.communicate(), timeout=_HARNESS_CHECK_TIMEOUT_S + ) + except TimeoutError: + proc.kill() + await proc.wait() + _logger.warning( + "Harness compile check skipped: forge build exceeded %ds", _HARNESS_CHECK_TIMEOUT_S + ) + return None + + try: + report = ForgeReport.model_validate_json(stdout_b.decode()) + except ValidationError: + _logger.warning( + "Harness compile check skipped: forge emitted no report: %s", stderr_b.decode()[-500:] + ) + return None + if not report.compile_errors: + return None + return "\n".join(report.compile_errors) + + class GeneratedHarness(BaseModel): """A generated harness file that creates a uniquely-named contract extending an external contract.""" path: str = Field(description="Path to the harness definition") @@ -215,7 +365,6 @@ class HarnessAgentResult(BaseModel): "A map from each target contract's `solidity_identifier` (exactly as given in " "the input list) to the harnesses chosen for it." )) - solidity_compiler: str = Field(description=f"The solidity compiler to use for compiling these harnesses.") class HarnessResult(BaseModel): identifier_to_source: dict[SolidityIdentifier, list[GeneratedHarnessSource]] @@ -230,10 +379,17 @@ class HarnessGenParams(TypedDict): _HarnessGenerationPrompt = TypedTemplate[HarnessGenParams]("harness_generation_prompt.j2") -def harness_generation_key( - instructions: AgentSystemDescription -) -> CacheKey[SystemDescriptionHarnessed, HarnessResult]: - return CacheKey[SystemDescriptionHarnessed, HarnessResult](string_hash(instructions.model_dump_json())) +def _system_setup_key(s: SourceApplication) -> str: + return "system-setup-" + string_hash(s.model_dump_json()) + +SYSTEM_SETUP_KEY = KeyFamily(ContractSetup, SystemDescriptionHarnessed, _system_setup_key) + +def _harness_generation_key(instructions: AgentSystemDescription) -> str: + return string_hash(instructions.model_dump_json()) + +HARNESS_GENERATION_KEY = KeyFamily( + SystemDescriptionHarnessed, HarnessResult, _harness_generation_key +) async def generate_harnesses( context: WorkflowContext[SystemDescriptionHarnessed], @@ -242,7 +398,7 @@ async def generate_harnesses( application: SourceApplication, instructions: AgentSystemDescription ) -> HarnessResult: - child = await context.child(harness_generation_key(instructions), instructions.model_dump()) + child = await context.child(HARNESS_GENERATION_KEY(instructions), instructions.model_dump()) if (cached := await child.cache_get(HarnessResult)) is not None: return cached @@ -284,48 +440,42 @@ class GenerationInput(FlowInput, VFSState): } - def result_validator( - s: GenerationState, - res: HarnessAgentResult, - tid: str - ) -> str | None: - check_copy = expected.copy() - all_files = [ - - ] - for (nm, r) in res.identifier_to_source.items(): - if nm not in check_copy: - return f"Delivered result for contract {nm}, but no instructions were given to harness it" - if len(r) != check_copy[nm]: - return f"Delivered {len(r)} harnesses for {nm}, but {check_copy[nm]} were required" - for res_c in r: - if mat.get(s, res_c.path) is None: - return f"Delivered harness {res_c.harness_name} at {res_c.path} for {nm}, but it doesn't exist on the VFS" - all_files.append(res_c.path) - del check_copy[nm] - if len(check_copy) != 0: - error = ", ".join( - [ f"contract {k} ({n} copies)" for (k,n) in check_copy.items() ] - ) - return f"Missing harnesses in results: {error}" - if False: # this doesn't work - with mat.materialize(s) as temp_dir: - compile_result = subprocess.run( - [res.solidity_compiler] + all_files, - cwd=temp_dir, - capture_output=True, - text=True + class HarnessResultTool(AsyncResultTool[HarnessAgentResult], WithInjectedState[GenerationState]): + """Signal the completion of your workflow. Triggers a compile of the + delivered harnesses against the project; a compile failure is reported + back to you for a retry. + """ + + @override + async def validate_result(self, res: HarnessAgentResult) -> str | None: + check_copy = expected.copy() + harness_paths: list[str] = [] + for (nm, r) in res.identifier_to_source.items(): + if nm not in check_copy: + return f"Delivered result for contract {nm}, but no instructions were given to harness it" + if len(r) != check_copy[nm]: + return f"Delivered {len(r)} harnesses for {nm}, but {check_copy[nm]} were required" + for res_c in r: + if mat.get(self.state, res_c.path) is None: + return f"Delivered harness {res_c.harness_name} at {res_c.path} for {nm}, but it doesn't exist on the VFS" + harness_paths.append(res_c.path) + del check_copy[nm] + if len(check_copy) != 0: + error = ", ".join( + [ f"contract {k} ({n} copies)" for (k,n) in check_copy.items() ] ) - if compile_result.returncode != 0 and False: - return f"Harness compilation failed:\nstdout:\n{compile_result.stdout}\nstderr:\n{compile_result.stderr}" - return None - - result_tool = result_tool_generator( - "result", - HarnessAgentResult, - "Signal the completion of your workflow", - validator=(GenerationState, result_validator) - ) + return f"Missing harnesses in results: {error}" + # Compile the agent's view of the project, not the project on disk: the + # harnesses it wrote live on the VFS, and a helper it wrote but did not + # deliver still has to be there for their imports to resolve. + async with _materialized(mat, self.state) as project_dir: + compile_errors = await _compile_check(project_dir, harness_paths) + if compile_errors is not None: + return ( + "The delivered harnesses do not compile. Repair them and deliver again; " + "the compiler reported:\n" + compile_errors + ) + return None g = env.builder_lite().with_input( GenerationInput @@ -338,7 +488,7 @@ def result_validator( ).with_sys_prompt_template( "harness_generation_system_prompt.j2" ).with_tools( - v_tools + [result_tool] + v_tools + [HarnessResultTool.as_tool("result")] ).with_default_summarizer().compile_async() res_state = await run_to_completion( @@ -436,7 +586,7 @@ async def run_setup_part1( env: ServiceHost, application_desc: SourceApplication ) -> SystemDescriptionHarnessed: - setup_ctx = await context.child(system_setup_key(application_desc), application_desc.model_dump()) + setup_ctx = await context.child(SYSTEM_SETUP_KEY(application_desc), application_desc.model_dump()) if (cached := await setup_ctx.cache_get(SystemDescriptionHarnessed)): return cached @@ -513,9 +663,6 @@ async def run_and_apply_part1( config_key = CacheKey[None, ContractSetup]("config") -from logging import getLogger -_logger = getLogger(__name__) - # --------------------------------------------------------------------------- # Split phases. @@ -523,7 +670,7 @@ async def run_and_apply_part1( # Harness creation and AutoSetup are exposed as two separate, independently # cached steps so the pipeline can run AutoSetup in parallel with invariant/bug # analysis. They share the ``config_key`` parent context, so existing -# harness-creation caches (keyed by ``system_setup_key``) still hit; the +# harness-creation caches (keyed by ``SYSTEM_SETUP_KEY``) still hit; the # AutoSetup result is cached under its own key. # --------------------------------------------------------------------------- @@ -541,19 +688,19 @@ async def run_harness_creation( return await run_and_apply_part1(config_ctxt, source, env, application_desc) -def autosetup_key( +def _autosetup_key( app: SourceApplication, prover_opts: ProverOptions, -) -> CacheKey[ContractSetup, SetupSuccess]: - """Cache key for the AutoSetup phase. Includes ``prover_opts`` so cloud and - local configurations never collide (the old composite ``config_key`` omitted - them, which could reuse a stale config across modes).""" - return CacheKey[ContractSetup, SetupSuccess]( - "autosetup-" + string_hash( - app.model_dump_json() + "\x00" + "\x00".join(prover_opts.extra_args) - ) +) -> str: + return "autosetup-" + string_hash( + app.model_dump_json() + "\x00" + "\x00".join(prover_opts.extra_args) ) +#: Cache key for the AutoSetup phase. Includes ``prover_opts`` so cloud and +#: local configurations never collide (the old composite ``config_key`` omitted +#: them, which could reuse a stale config across modes). +AUTOSETUP_KEY = KeyFamily(ContractSetup, SetupSuccess, _autosetup_key) + async def run_autosetup_phase( context: WorkflowContext[None], @@ -569,7 +716,7 @@ async def run_autosetup_phase( Cache hits are guarded by the on-disk existence of ``summaries_path``.""" config_ctxt = context.child(config_key) cache = await config_ctxt.child( - autosetup_key(application_desc, prover_opts), + AUTOSETUP_KEY(application_desc, prover_opts), application_desc.model_dump(), ) if (cached := await cache.cache_get(SetupSuccess)) is not None: diff --git a/composer/spec/source/keys.py b/composer/spec/source/keys.py new file mode 100644 index 00000000..b1c5acd5 --- /dev/null +++ b/composer/spec/source/keys.py @@ -0,0 +1,61 @@ +"""The prover (CVL) backend's cache sub-chain, declared in one place. + +Everything the prover backend adds around the Auto-Prove driver chain +(``composer.pipeline.keys``), in tree order:: + + run root + ├── config config_key → ContractSetup + │ ├── system-setup-{app digest} SYSTEM_SETUP_KEY → SystemDescriptionHarnessed + │ │ ├── harness-analysis HARNESS_ANALYSIS_KEY → AgentSystemDescription + │ │ └── {instructions digest} HARNESS_GENERATION_KEY → HarnessResult + │ └── autosetup-{app+opts digest} AUTOSETUP_KEY → SetupSuccess + ├── summary-{config digest} SUMMARY_KEY → _SummaryCache + ├── structural-inv STRUCTURAL_INV_KEY → Invariants + │ └── judge INV_JUDGE_KEY → (judge memory) + ├── invariant-cvl INV_CVL_KEY → GeneratedCVL + │ ├── judge CVL_JUDGE_KEY → (judge memory) + │ └── last_attempt LAST_ATTEMPT_KEY → _LastAttemptCache + └── ap-properties PROPERTIES_KEY(AP_PROPERTIES_KEY_NAME) + └── … driver chain … + └── {props digest} FORMALIZATION_KEY → GeneratedCVL + ├── judge CVL_JUDGE_KEY + └── last_attempt LAST_ATTEMPT_KEY + +Families and constants are declared beside their cache models (harness, +summarizer, struct_invariant, cvl_generation) and gathered here; +consumers (the backend pipeline, ``cache-autoprove``) import from this +registry rather than spelunking the producer modules. +""" + +from composer.spec.context import CacheKey +from composer.spec.cvl_generation import ( + CVL_JUDGE_KEY, LAST_ATTEMPT_KEY, GeneratedCVL, +) +from composer.spec.source.harness import ( + AUTOSETUP_KEY, HARNESS_ANALYSIS_KEY, HARNESS_GENERATION_KEY, + SYSTEM_SETUP_KEY, config_key, +) +from composer.spec.source.struct_invariant import INV_JUDGE_KEY, STRUCTURAL_INV_KEY +from composer.spec.source.summarizer import SUMMARY_KEY + +__all__ = [ + "AP_PROPERTIES_KEY_NAME", + "AUTOSETUP_KEY", + "CVL_JUDGE_KEY", + "HARNESS_ANALYSIS_KEY", + "HARNESS_GENERATION_KEY", + "INV_CVL_KEY", + "INV_JUDGE_KEY", + "LAST_ATTEMPT_KEY", + "STRUCTURAL_INV_KEY", + "SUMMARY_KEY", + "SYSTEM_SETUP_KEY", + "config_key", +] + +#: The prover backend's ``SystemAnalysisSpec.properties_key``. +AP_PROPERTIES_KEY_NAME = "ap-properties" + +#: CVL generation for the structural invariants (the per-component peer is +#: reached via ``FORMALIZATION_KEY``). +INV_CVL_KEY = CacheKey[None, GeneratedCVL]("invariant-cvl") diff --git a/composer/spec/source/live_explorer.py b/composer/spec/source/live_explorer.py index 4577cb45..30dae070 100644 --- a/composer/spec/source/live_explorer.py +++ b/composer/spec/source/live_explorer.py @@ -17,7 +17,11 @@ from composer.spec.agent_index import AgentIndex, RetrieveDocumentTool from composer.spec.source.versioned_index import VersionedAgentIndex, MigrationOracle -from composer.spec.code_explorer import _ExploreCodeCommon, CODE_EXPLORER_SYS_PROMPT +from composer.pipeline.ecosystem import Ecosystem +from composer.spec.code_explorer import ( + _ExploreCodeCommon, + code_explorer_sys_prompt, +) from composer.spec.context import SourceCode, user_data_ns from composer.spec.util import uniq_thread_id @@ -119,27 +123,6 @@ class LiveEditTools: class _LiveExplorerState(MessagesState, VFSState): result: NotRequired[str] -_VERSIONED_INDEXED_SYS_PROMPT = CODE_EXPLORER_SYS_PROMPT + """ - -You may be provided with other question/answer pairs that were found to be similar -to the question you are asked. These question/answer pairs *may* have been derived -on a prior version of the codebase that you are exploring now; such pairs will be clearly -marked as being (potentially) out of date. Use the following protocol to use these -prior results effectively: - -1. If a prior finding is *not* marked as out of date, and directly answers the question you are asked, - use that answer as is; do not rephrase, re-investigate, or "verify" the answer -2. If a prior finding is *not* marked as out of date, and *partially* answers the question you are asked, - use that answer as a verified starting point and fill in any missing details. - -If a prior question/answer pair that is marked as (potentially stale) -either completely or partially answers the question posed to you, you *should* -use your source tools to determine if the substantive and relevant details of the answer -are still true on this version of the code. If you verify that these details -remain true, you may reuse (in part or in whole) the existing answer as you would -an up-to-date answer. -""" - def _prover_output_dirs(p: PurePath) -> bool: """Globally exclude prover outputs from the live tool surface AND the materializer: they are never compilation inputs, and copying prior runs' @@ -157,7 +140,8 @@ def setup_live_edits( store: BaseStore, source_key: str, oracle: MigrationOracle, - recursion_limit: int + recursion_limit: int, + ecosystem: Ecosystem, ) -> LiveEditTools: x = VersionedAgentIndex( _wrapped=base_store, @@ -195,7 +179,7 @@ def setup_live_edits( ) ]) .with_initial_prompt("Answer the following question") - .with_sys_prompt(_VERSIONED_INDEXED_SYS_PROMPT) + .with_sys_prompt(code_explorer_sys_prompt(ecosystem.code_explorer_prompt, "versioned")) .with_output_key("result") .compile_async() ) diff --git a/composer/spec/source/monitor.py b/composer/spec/source/monitor.py new file mode 100644 index 00000000..433c8304 --- /dev/null +++ b/composer/spec/source/monitor.py @@ -0,0 +1,21 @@ +from typing import TYPE_CHECKING + +from langgraph.config import get_config +from langchain_core.messages import HumanMessage +from composer.spec.graph_builder import run_to_completion +from graphcore.graph import MonitorReturn +from composer.prover.ptypes import StatusCodes + +if TYPE_CHECKING: + # Runtime import would be circular: author.py imports this module. + from .author import SourceCVLGenerationState + +def monitor( + curr_state: "SourceCVLGenerationState" +) -> MonitorReturn: + if not curr_state["reminders_channel"]: + return None, None + + return [HumanMessage(f"{'\n'.join(curr_state["reminders_channel"])}")], { + "reminders_channel": [] + } diff --git a/composer/spec/source/munge/edit_store.py b/composer/spec/source/munge/edit_store.py index 511e0b61..3d323963 100644 --- a/composer/spec/source/munge/edit_store.py +++ b/composer/spec/source/munge/edit_store.py @@ -4,15 +4,53 @@ from langgraph.store.base import BaseStore +@dataclass(frozen=True) +class MungeEditor: + """The munge editor sub-agent, commissioned through the author's + edit-request tool. Every record written before attribution existed has + this provenance — it was the only committer.""" + + +@dataclass(frozen=True) +class PluginEditor: + """A pipeline plugin proposed this edit through the edit store staged into + its contributed tools; the CVL author still decides whether to apply it.""" + plugin: str + + +type EditAttribution = MungeEditor | PluginEditor + + +def _attribution_payload(a: EditAttribution) -> dict: + match a: + case MungeEditor(): + return {"kind": "munge-editor"} + case PluginEditor(plugin=plugin): + return {"kind": "plugin", "plugin": plugin} + + +def _attribution_of(v: object) -> EditAttribution: + match v: + case None: + return MungeEditor() + case {"kind": "munge-editor"}: + return MungeEditor() + case {"kind": "plugin", "plugin": str(plugin)}: + return PluginEditor(plugin=plugin) + case _: + raise ValueError(f"Unrecognized edit attribution payload: {v!r}") + + @dataclass(frozen=True) class StoredEdit: - """A committed edit: the full VFS snapshot plus the editor's account of it. - The description fields ride into the edit-history log and the final - deliverable, so a reader can tell why the source was changed without - reconstructing the diff.""" + """A committed edit: the full VFS snapshot, the committer's account of it, + and who committed it. The description fields ride into the edit-history log + and the final deliverable, so a reader can tell why the source was changed + without reconstructing the diff; ``attribution`` says on whose authority.""" vfs: dict[str, str] executive_summary: str why_sound: str + attribution: EditAttribution @dataclass @@ -29,6 +67,7 @@ async def read(self, id: str) -> StoredEdit | None: vfs=cast(dict[str, str], v["vfs"]), executive_summary=cast(str, v["executive_summary"]), why_sound=cast(str, v["why_sound"]), + attribution=_attribution_of(v.get("attribution")), ) @classmethod @@ -41,12 +80,14 @@ def _deterministic_hash(cls, vfs: dict[str, str]) -> str: return hasher.hexdigest() async def commit( - self, vfs: dict[str, str], *, executive_summary: str, why_sound: str + self, vfs: dict[str, str], *, executive_summary: str, why_sound: str, + attribution: EditAttribution, ) -> str: id = self._deterministic_hash(vfs) await self._store.aput(self._target_ns, id, { "vfs": {**vfs}, "executive_summary": executive_summary, "why_sound": why_sound, + "attribution": _attribution_payload(attribution), }) return id diff --git a/composer/spec/source/munge/munge_agent.py b/composer/spec/source/munge/munge_agent.py index 1320ee29..eb290968 100644 --- a/composer/spec/source/munge/munge_agent.py +++ b/composer/spec/source/munge/munge_agent.py @@ -10,7 +10,7 @@ from langchain_core.tools import BaseTool from langchain_core.messages import ToolMessage -from .edit_store import EditStore +from .edit_store import EditStore, MungeEditor from .vfs_diff import summarize_changes from composer.spec.source.conf_maps import ( CompilerSettings, MapViolations, extend_compiler_maps, map_violation_message, @@ -146,6 +146,7 @@ async def run(self) -> str: res["vfs"], executive_summary=d.executive_summary, why_sound=d.why_sound, + attribution=MungeEditor(), ) diff = summarize_changes( res, deps.accessor, self.state["vfs"] diff --git a/composer/spec/source/pipeline.py b/composer/spec/source/pipeline.py index 93ee3416..e5a80c14 100644 --- a/composer/spec/source/pipeline.py +++ b/composer/spec/source/pipeline.py @@ -22,13 +22,13 @@ import asyncio from dataclasses import dataclass from pathlib import Path -from typing import override +from typing import override, Sequence from langchain_core.tools import BaseTool from composer.io.multi_job import TaskInfo -from composer.spec.context import WorkflowContext, CacheKey, CVLGeneration -from composer.spec.types import PropertyFormulation +from composer.spec.context import WorkflowContext, CVLGeneration +from composer.spec.types import PropertyFormulation, PropertyTitle from composer.spec.gen_types import CVLResource, SPECS_DIR, certora_relative_to_project from composer.spec.system_model import ( ContractComponentInstance, ContractInstance, SourceApplication, HarnessedApplication, @@ -39,20 +39,23 @@ from composer.spec.prop_inference import CERTORA_BACKEND_GUIDANCE from composer.spec.source.harness import ( run_harness_creation, run_autosetup_phase, ContractSetup, SystemDescriptionHarnessed, + lift_harnessed, ) from composer.spec.source.summarizer import setup_summaries from composer.spec.source.struct_invariant import get_invariant_formulation from composer.spec.source.autosetup import SetupSuccess from composer.spec.source.prover import get_prover_tool, materializing_project -from composer.spec.source.author import batch_cvl_generation, SourceEditing +from composer.spec.source.plugin import CertoraProverTools +from composer.spec.source.author import batch_cvl_generation, EditingTools, SourceEditing, ProverTool from composer.spec.source.artifacts import ProverArtifactStore, ComponentSpec, InvariantSpec from composer.spec.source.report_prover import make_prover_fetcher from composer.spec.source.report.collect import ( - EvidenceFetcher, ReportComponentInput, RuleEvidence, Verdict, VerdictFetcher, + Formalized, EvidenceFetcher, ReportComponentInput, RuleEvidence, Verdict, VerdictFetcher, +) +from composer.spec.source.report.schema import ( + AppliedEditRecord, ComponentName, RuleName, SourceEditRecord, ) -from composer.spec.source.report.schema import RuleName from composer.spec.source.cex_capture import CexAnalysisStore -from composer.spec.source.report.schema import AppliedEditRecord, RuleName, SourceEditRecord from composer.spec.source.munge.vfs_diff import diff_against_baseline from composer.spec.source.task_ids import ( @@ -63,46 +66,26 @@ from composer.ui.autoprove_app import AutoProvePhase from composer.pipeline.core import ( Formalizer, PreparedSystem, PipelineRun, Delivered, GaveUp, - CorePhases, SystemAnalysisSpec, ComponentOutcome, - COMMON_SYSTEM_CACHE_KEY + CorePhases, SystemAnalysisSpec, ComponentOutcome, ToolBinder, + Curtailed ) from composer.pipeline.ecosystem import main_instance +from composer.pipeline.keys import COMMON_SYSTEM_CACHE_KEY +from composer.spec.source.keys import AP_PROPERTIES_KEY_NAME, INV_CVL_KEY +@dataclass +class _ProverPipelineDeps: + prover_options: ProverOptions + store: ProverArtifactStore + analysis_store: CexAnalysisStore + editing: SourceEditing -INV_CVL_KEY = CacheKey[None, GeneratedCVL]("invariant-cvl") - - -def _lift_harnessed( - s: SourceApplication, sys_desc: SystemDescriptionHarnessed, -) -> HarnessedApplication: - """Re-key harness definitions by harnessed contract and fold them into a - ``HarnessedApplication`` — each ``SourceExplicitContract`` becomes a - ``HarnessedExplicitContract`` carrying the harnesses generated for it.""" - contract_to_harness: dict[SolidityIdentifier, list[HarnessDefinition]] = {} - for c in sys_desc.transitive_closure: - if not c.harness_definition: - continue - contract_to_harness.setdefault(c.harness_definition.harness_of, []).append( - HarnessDefinition(name=c.solidity_identifier, path=c.path) - ) + def to_prover_tool(self, tool: BaseTool) -> ProverTool: + return ProverTool(lg_tool=tool, options=self.prover_options) - comp: list[SourceExternalActor | HarnessedExplicitContract] = [] - for c in s.components: - if not isinstance(c, SourceExplicitContract): - comp.append(c) - continue - comp.append(HarnessedExplicitContract( - sort=c.sort, - name=c.name, - solidity_identifier=c.solidity_identifier, - components=c.components, - description=c.description, - path=c.path, - harnesses=contract_to_harness.get(c.solidity_identifier, []), - )) - return HarnessedApplication( - application_type=s.application_type, description=s.description, components=comp, - ) +#: The invariant CVL's slot in the report: a real delivery (imported by every component spec) +#: or the quarantined leftovers of a budget-curtailed generation (appendix only). +type InvariantResult = Delivered[GeneratedCVL] | Curtailed[Delivered[GeneratedCVL]] @dataclass @@ -110,14 +93,14 @@ class ProverRunner(Formalizer[GeneratedCVL, ContractComponentInstance]): """Immutable formalizer: per-batch CVL generation against a fixed prover config + resource set (already including ``invariants.spec`` when there are structural invariants), plus the in-memory invariant result for the report.""" - _store: ProverArtifactStore _prover_tool: BaseTool _prover_config: dict _resources: list[CVLResource] - _invariant: tuple[list[PropertyFormulation], Delivered[GeneratedCVL]] | None + _invariant: tuple[list[PropertyFormulation], InvariantResult] | None _fetch: VerdictFetcher[GeneratedCVL] - _editing: SourceEditing - _analysis_store: CexAnalysisStore + _deps: _ProverPipelineDeps + + tool_provider_type = CertoraProverTools @override async def formalize( @@ -127,20 +110,23 @@ async def formalize( props: list[PropertyFormulation], ctx: WorkflowContext[GeneratedCVL], run: PipelineRun, - ) -> GeneratedCVL | GaveUp: + extra_tools: ToolBinder[ContractComponentInstance] + ) -> GeneratedCVL | Curtailed[GeneratedCVL] | GaveUp: return await batch_cvl_generation( ctx=ctx.abstract(CVLGeneration), init_config=self._prover_config, props=props, component=feat, resources=self._resources, - prover_tool=self._prover_tool, + prover_tool=self._deps.to_prover_tool(self._prover_tool), env=run.env, description=label, source=run.source, spec_dir=SPECS_DIR, spec_stem=ComponentSpec(feat.slugified_name).stem, - editing=self._editing, + editing_tools=EditingTools( + editing=self._deps.editing, tool_provider=extra_tools + ), ) @override @@ -150,14 +136,14 @@ def extra_report_inputs(self) -> list[ReportComponentInput[GeneratedCVL]]: return [] inv_props, inv = self._invariant return [ReportComponentInput( - name="Structural Invariants", props=inv_props, formalized=inv, + name=ComponentName("Structural Invariants"), props=inv_props, formalized=inv, )] @override async def fetch_verdicts( - self, inp: ReportComponentInput[GeneratedCVL], + self, formalized: Formalized[GeneratedCVL], ) -> dict[RuleName, Verdict]: - return await self._fetch(inp) + return await self._fetch(formalized) @override async def source_edits( @@ -189,12 +175,13 @@ async def _fetch_evidence(self, rule_name: str) -> list[RuleEvidence]: # per binding while the report shows a single row for the whole rule. return [ RuleEvidence(label=r.label, analysis=r.analysis, counterexample=r.counterexample) - for r in await self._analysis_store.for_rule(rule_name) + for r in await self._deps.analysis_store.for_rule(rule_name) ] @override async def finalize(self, outcomes: list[ComponentOutcome[GeneratedCVL, ContractComponentInstance]], run: PipelineRun) -> None: - # components_to_prover_runs.json: {run_key (slug): prover /output/ link}. + # components_to_prover_runs.json: {run_key (slug): prover /output/ link}. Deliveries + # only — a curtailed partial's last run says nothing about its final content. runs: dict[str, str] = { ComponentSpec(o.feat.slugified_name).run_key: o.result.run_link for o in outcomes @@ -202,23 +189,21 @@ async def finalize(self, outcomes: list[ComponentOutcome[GeneratedCVL, ContractC } if self._invariant is not None: inv = self._invariant[1] - if inv.run_link: + if isinstance(inv, Delivered) and inv.run_link: runs[InvariantSpec().run_key] = inv.run_link - self._store.write_component_runs(runs) + self._deps.store.write_component_runs(runs) @dataclass class ProverPrepared(PreparedSystem[GeneratedCVL, ContractComponentInstance, ContractInstance]): """Post-harness system: holds the harnessed app + prover tool, and runs the prover-only pre-formalization fan-out in ``prepare_formalization``.""" - _store: ProverArtifactStore _sys_desc: SystemDescriptionHarnessed _harnessed: HarnessedApplication _prover_tool: BaseTool - _prover_opts: ProverOptions _analyzed: SourceApplication - _editing: SourceEditing - _analysis_store: CexAnalysisStore + + _deps: _ProverPipelineDeps @override async def prepare_formalization(self, run: PipelineRun) -> Formalizer[GeneratedCVL, ContractComponentInstance]: @@ -228,16 +213,19 @@ async def prepare_formalization(self, run: PipelineRun) -> Formalizer[GeneratedC self._autosetup(run), self._invariants(run), ) - invariant: tuple[list[PropertyFormulation], Delivered[GeneratedCVL]] | None = None + invariant: tuple[list[PropertyFormulation], InvariantResult] | None = None if invariants.inv: inv_props = [ - PropertyFormulation(title=inv.name, description=inv.description, sort="invariant") + PropertyFormulation( + title=PropertyTitle(inv.name), description=inv.description, sort="invariant", + ) for inv in invariants.inv ] - self._store.write_properties(InvariantSpec(), inv_props) + self._deps.store.write_properties(InvariantSpec(), inv_props) inv_cvl_ctx = run.ctx.child(INV_CVL_KEY) cached = await inv_cvl_ctx.cache_get(GeneratedCVL) + inv_cvl: GeneratedCVL | Curtailed[GeneratedCVL] if cached is not None: inv_cvl = cached else: @@ -249,7 +237,7 @@ async def prepare_formalization(self, run: PipelineRun) -> Formalizer[GeneratedC props=inv_props, component=None, resources=resources, - prover_tool=self._prover_tool, + prover_tool=self._deps.to_prover_tool(self._prover_tool), env=run.env, description="Structural invariant CVL", source=run.source, @@ -258,8 +246,9 @@ async def prepare_formalization(self, run: PipelineRun) -> Formalizer[GeneratedC # Invariants are assumed as preconditions by every # downstream spec, so they must hold against the # unedited source: no editor, frozen source tools, - # immutable-source judge. - editing=None, + # immutable-source judge — and, since the two travel + # together, no plugin tool contribution either. + editing_tools=None, ), ) if isinstance(inv_result, GaveUp): @@ -267,31 +256,46 @@ async def prepare_formalization(self, run: PipelineRun) -> Formalizer[GeneratedC f"Structural invariant CVL generation gave up: {inv_result.reason}" ) inv_cvl = inv_result - await inv_cvl_ctx.cache_put(inv_cvl) - - # Writes invariants.spec + bundle, returns its project-root-relative path. - inv_path = self._store.write_artifact(InvariantSpec(), inv_cvl) - # All pre-formalization work has joined, so appending here is race-free; - # the per-component CVLs (run after this returns) will see invariants.spec. - resources = [*resources, CVLResource( - path=inv_path, - required=False, - description="Structural invariants that may be assumed as preconditions", - sort="import", - )] - invariant = (inv_props, Delivered(inv_cvl, inv_path)) + if isinstance(inv_result, GeneratedCVL): + await inv_cvl_ctx.cache_put(inv_result) + + if isinstance(inv_cvl, Curtailed): + # The budget cut the invariant CVL short. An unreliable invariants.spec must not + # be imported into the per-component specs as assumed preconditions, so the + # partial (if any) is quarantined for inspection and the invariants surface only + # in the report's budget appendix; the run itself degrades gracefully. + partial = ( + Delivered( + inv_cvl.partial, + self._deps.store.write_quarantined(InvariantSpec(), inv_cvl.partial), + ) + if inv_cvl.partial is not None else None + ) + invariant = (inv_props, Curtailed(partial, inv_cvl.detail)) + else: + # Writes invariants.spec + bundle, returns its project-root-relative path. + inv_path = self._deps.store.write_artifact(InvariantSpec(), inv_cvl) + # All pre-formalization work has joined, so appending here is race-free; + # the per-component CVLs (run after this returns) will see invariants.spec. + resources = [*resources, CVLResource( + path=inv_path, + required=False, + description="Structural invariants that may be assumed as preconditions", + sort="import", + )] + invariant = (inv_props, Delivered(inv_cvl, inv_path)) return ProverRunner( GeneratedCVL, "prover", - self._store, self._prover_tool, setup_config.prover_config, resources, invariant, - make_prover_fetcher(), self._editing, self._analysis_store, + self._prover_tool, setup_config.prover_config, resources, invariant, + make_prover_fetcher(), self._deps ) async def _autosetup(self, run: PipelineRun) -> tuple[SetupSuccess, list[CVLResource]]: setup_config = await run.runner( TaskInfo(AUTOSETUP_TASK_ID, "AutoSetup", AutoProvePhase.AUTOSETUP), lambda: run_autosetup_phase( - run.ctx, run.source, self._sys_desc, self._analyzed, self._prover_opts, + run.ctx, run.source, self._sys_desc, self._analyzed, self._deps.prover_options, ), ) resources: list[CVLResource] = [CVLResource( @@ -320,13 +324,10 @@ async def _invariants(self, run: PipelineRun): lambda: get_invariant_formulation(run.ctx, run.source, run.env, self._harnessed), ) -AP_PROPERTIES_KEY_NAME = "ap-properties" - @dataclass class ProverBackend: - """PipelineBackend[AutoProvePhase, GeneratedCVL, None, ComponentSpec, - ContractComponentInstance, ContractInstance, SourceApplication] - (P, FormT, H, A, Unit, Main, App).""" + """``PipelineBackend[AutoProvePhase, GeneratedCVL, None, SpecIdentity, ContractComponentInstance, + ContractInstance, SourceApplication, None]`` (P, FormT, H, A, Unit, Main, App, Pre) — structural.""" backend_guidance = CERTORA_BACKEND_GUIDANCE core_phases = CorePhases({ "analysis": AutoProvePhase.COMPONENT_ANALYSIS, @@ -341,14 +342,21 @@ class ProverBackend: editing: SourceEditing analysis_store: CexAnalysisStore + async def preflight(self, run: PipelineRun[AutoProvePhase, None]) -> None: + """Nothing to do ahead of analysis. The prover's expensive pre-work (AutoSetup, summaries, + structural invariants) needs the *harnessed* model, so it stays in ``prepare_formalization``, + where it already overlaps property extraction.""" + return None + async def prepare_system( self, analyzed: SourceApplication, run: PipelineRun[AutoProvePhase, None], + preflight: None, ) -> PreparedSystem[GeneratedCVL, ContractComponentInstance, ContractInstance]: sys_desc = await run.runner( TaskInfo(HARNESS_TASK_ID, "Harness Creation", AutoProvePhase.HARNESS), lambda: run_harness_creation(run.ctx, run.source, run.env, analyzed), ) - harnessed = _lift_harnessed(analyzed, sys_desc) + harnessed = lift_harnessed(analyzed, sys_desc) # The materializing strategy covers every phase with one tool: an empty # VFS (invariants, or an author that never edited) runs in-situ; a # non-empty one runs in a temp materialization of the working copy. @@ -357,9 +365,10 @@ async def prepare_system( prover_opts=self._prover_opts, analysis_store=self.analysis_store, ) return ProverPrepared( - main_instance(harnessed, run.source), - self.artifact_store, sys_desc, harnessed, prover_tool, - self._prover_opts, analyzed, self.editing, self.analysis_store, + main=main_instance(harnessed, run.source), + _sys_desc=sys_desc, _harnessed=harnessed, _prover_tool=prover_tool, + _analyzed=analyzed, + _deps=_ProverPipelineDeps(self._prover_opts, self.artifact_store, self.analysis_store, self.editing) ) def to_artifact_id(self, c: ContractComponentInstance) -> ComponentSpec: diff --git a/composer/spec/source/plugin.py b/composer/spec/source/plugin.py new file mode 100644 index 00000000..b54eac1d --- /dev/null +++ b/composer/spec/source/plugin.py @@ -0,0 +1,91 @@ +"""The prover backend's plugin-extension surface. + +What a plugin imports to contribute tools to the prover's CVL author: the +:class:`CertoraProverTools` provider class (deriving it IS the declaration — see +``composer.pipeline.plugin_api``) and the :class:`ProverState` view its hooks can +stage at tool-invocation time. Lives here rather than in the plugin API so the +core/plugin layers stay backend-agnostic; the author hands the provider class to +the driver's binder as a ``ToolExtension`` (``composer.pipeline.core``). +""" + +import pathlib +from abc import abstractmethod +from dataclasses import dataclass +from typing import AsyncContextManager, Callable, Sequence, Protocol + +from composer.pipeline.plugin_api import ( + FormalizationTool, PipelinePlugin, PluginToolContext, ProvidedTools, +) +from composer.spec.system_model import FeatureUnit +from composer.spec.types import PropertyFormulation +from composer.prover.core import ProverReport, ProverCallbacks, CexHandler +from composer.io.task_host import TaskHost + +class ProverRunner(Protocol): + """One ad-hoc prover run: stages the spec/conf into ``working_dir`` for the + duration of the call and forwards to ``run_prover``. ``config`` entries + override the author's current prover config for this run only.""" + async def __call__( + self, + *, + curr_spec: str, + working_dir: str, + cex_handler: CexHandler, + callbacks: ProverCallbacks, + tool_call_id: str, + rules: list[str] | None = None, + **config, + ) -> ProverReport | str: + ... + +class EditProposer(Protocol): + """A plugin's narrowed window onto the run's edit store: stage a source + edit for the CVL author to consider. Proposing only records the edit and + returns its id — nothing changes until the author applies that id through + its edit-management tools. The implementation stamps the proposing + plugin's identity onto the record; attribution is not the proposer's to + choose. + + ``vfs`` is the proposer's overlay *relative to the staged working copy* + its :class:`CVLAuthorState` was read with (the only view a plugin has). + The implementation completes it into the edit store's full-snapshot form — + merging the overlay the staged copy was materialized from — before + committing.""" + + async def propose( + self, vfs: dict[str, str], *, executive_summary: str, why_sound: str + ) -> str: + ... + + +@dataclass +class CVLAuthorState: + # The run root the prover would execute in: the project itself, or a temporary + # materialization of the author's working copy (lifetime = the read). + working_dir: pathlib.Path + curr_spec: str | None + prover_runner: ProverRunner + host: TaskHost + # Propose source edits for the author to apply; records carry the + # proposing plugin's attribution. + edit_store: EditProposer + +type ProverStateReader[T] = Callable[[T], AsyncContextManager[CVLAuthorState]] + + +class CertoraProverTools[U: FeatureUnit](PipelinePlugin[U]): + """Contributes tools to the prover's CVL author. ``st``/``state_reader`` let a + contributed tool stage the author's live dependencies (:class:`ProverState`) at + invocation time: bind the reader into the tool, and open it against the injected + graph state of type ``st``.""" + + @abstractmethod + async def certora_prover_tools[T]( + self, + comp: U, + prop: Sequence[PropertyFormulation], + tool_context: PluginToolContext[FormalizationTool], + state_reader: ProverStateReader[T], + st: type[T], + ) -> ProvidedTools | None: + ... diff --git a/composer/spec/source/prover.py b/composer/spec/source/prover.py index 373ea0ce..7dcf1873 100644 --- a/composer/spec/source/prover.py +++ b/composer/spec/source/prover.py @@ -15,7 +15,8 @@ from contextlib import contextmanager, asynccontextmanager, ExitStack, nullcontext from pathlib import Path from typing import ( - Annotated, AsyncIterator, Callable, Iterator, override, AsyncContextManager, + Annotated, AsyncIterator, Callable, Container, Iterable, Iterator, Mapping, override, + AsyncContextManager, Sequence, Literal ) from typing_extensions import TypedDict, NotRequired @@ -24,28 +25,32 @@ from composer.spec.source.live_explorer import VersionedHistory from langchain_core.tools import InjectedToolCallId, tool, BaseTool +from langchain_core.messages import AIMessage from langgraph.prebuilt import InjectedState -from pydantic import BaseModel, Field +from pydantic import BaseModel, Field, Discriminator from langgraph.config import get_stream_writer from langgraph.types import Command -from composer.prover.ptypes import RuleResult +from composer.prover.ptypes import RuleResult, RulePath from graphcore.graph import LLM from composer.prover.core import ( - ProverOptions, ProverCallbacks, run_prover, DefaultCexHandler + ProverOptions, declared_rules_list, run_prover, DefaultCexHandler ) from composer.prover.callbacks import ProverEventCallbacks +from composer.prover.ptypes import StatusCodes from composer.ui.tool_display import tool_display from composer.diagnostics.stream import ( ProverOutputEvent, CloudPollingEvent, RuleAnalysisResult, CEXAnalysisStart, ProverRun, ProverLink, ProverResult ) -from composer.spec.cvl_generation import CVLGenerationState, make_validation_stamper +from composer.authoring.state import make_validation_stamper, spec_digest +from composer.spec.cvl_generation import CVLGenerationState from composer.diagnostics.timing import RunSummary, get_run_summary from graphcore.graph import tool_state_update from composer.spec.util import temp_certora_file from composer.spec.gen_types import CERTORA_DIR, SPECS_DIR +from composer.spec.util import string_hash from composer.spec.source.cex_capture import CexAnalysisStore @@ -95,6 +100,155 @@ def _merge_rule_skips(left: dict[str, str], right: dict[str, str]) -> dict[str, to_ret[k] = v return to_ret +class RuleSelection(TypedDict): + sort: Literal["exclude", "include"] + selector: list[str] + +class ProverRunLog(TypedDict): + tool_call_id: str + prover_results: list[tuple[RulePath, StatusCodes]] + spec_digest: str + rules: RuleSelection | None + sort: Literal["run"] + declared_rules: list[str] + state_digest: str + +class NagMarker(TypedDict): + nagged_rules: list[RulePath] + sort: Literal["nag"] + +type ProverHistoryItem = Annotated[ProverRunLog | NagMarker, Discriminator("sort")] + +def _executed_rules( + r: ProverRunLog +) -> list[str]: + if r["rules"] is None: + return r["declared_rules"] + elif r["rules"]["sort"] == "include": + return r["rules"]["selector"] + else: + to_filt = set(r["rules"]["selector"]) + return [ r_id for r_id in r["declared_rules"] if r_id not in to_filt ] + +#: How many consecutive runs must end in the identical failure before the author is nagged +#: about a rule. Counts the run being processed, so 3 means "this run plus the two before it". +STUCK_RULE_NAG_THRESHOLD = 3 + + +def stuck_rule_warnings( + # Values are compared for equality only, so the looser ``str`` keeps callers free of + # the narrowing dance ``StatusCodes`` would otherwise force on a filtered comprehension. + stuck_rules: Mapping["RulePath", str], + prover_history: list[ProverHistoryItem], + known_tool_call_ids: Container[str | None], +) -> tuple[set["RulePath"], bool]: + """Decide which of the currently-stuck rules the author should be nagged about. + + Walks ``prover_history`` backwards counting, per stuck rule, how many *consecutive* + recent runs ended in the identical failure — each rule starts at 1 for the run being + processed. A rule leaves the tally as soon as a run breaks its streak (it either + passed, failed differently, or the tally is exhausted); reaching + :data:`STUCK_RULE_NAG_THRESHOLD` moves it to the returned warning set. A run that + targeted an explicit rule subset is transparent to rules it never exercised, so a + narrowly-scoped re-run neither extends nor breaks another rule's streak. + + A ``nag`` marker means those rules were warned about already, so their streaks restart + there and the author isn't nagged twice for the same stretch of failures. + + Returns the rules to warn about, plus whether any inspected run predates the author's + most recent history compaction (its tool call is no longer in ``known_tool_call_ids``) + — the caller footnotes the warning with that, since those runs are no longer visible + in the conversation. + """ + stuck_count = {k: 1 for k in stuck_rules} + to_warn: set[RulePath] = set() + seen_post_compaction_history = False + + history_ind = len(prover_history) - 1 + while history_ind >= 0 and len(stuck_count) > 0: + it = prover_history[history_ind] + history_ind -= 1 + if it["sort"] == "nag": + for r in it["nagged_rules"]: + # A previously-nagged rule need not be stuck now — drop it if present. + stuck_count.pop(r, None) + continue + assert it["sort"] == "run" + if it["tool_call_id"] not in known_tool_call_ids: + seen_post_compaction_history = True + target_rules = _executed_rules(it) + # Snapshot the keys: the body deletes from ``stuck_count`` as streaks end. + for k in list(stuck_count.keys()): + if k.rule not in target_rules: + continue + if not any( + rp == k and stuck_rules[k] == stat for (rp, stat) in it["prover_results"] + ): + del stuck_count[k] + else: + stuck_count[k] += 1 + if stuck_count[k] == STUCK_RULE_NAG_THRESHOLD: + to_warn.add(k) + del stuck_count[k] + return to_warn, seen_post_compaction_history + + +def last_prover_run( + l: list[ProverHistoryItem] +) -> ProverRunLog | None: + for i in range(len(l) - 1, -1, -1): + it = l[i] + if it["sort"] != "run": + continue + return it + return None + +def _iterate_history( + l: list[ProverHistoryItem], + curr_digest: str, + curr_status: list[tuple[RulePath, StatusCodes]], +) -> Iterable[list[tuple[RulePath, StatusCodes]]]: + """Newest-first walk of the prover results produced against the current authoring + state: the current run's results, then each prior run whose ``state_digest`` matches, + stopping at the first run recorded against a different state (nag markers are + transparent).""" + yield curr_status + for elem in reversed(l): + if elem["sort"] != "run": + continue + if elem["state_digest"] != curr_digest: + return + yield elem["prover_results"] + +def _is_completion_history( + l: list[ProverHistoryItem], + curr_digest: str, + expected_to_fail: set[str], + curr_status: list[tuple[RulePath, StatusCodes]], + all_rules: list[str] +) -> bool: + """Whether the runs against the current authoring state collectively verify every + declared rule (rules expected to fail are forgiven their failures but still count + as covered).""" + remaining_rules = set(all_rules) + for history in _iterate_history( + l, curr_digest, curr_status + ): + for (k, stat) in history: + if stat != "VERIFIED" and k.rule not in expected_to_fail: + return False + # discard, not remove: results can name rules outside the declared list + # (envfreeFuncsStaticCheck, parametric instantiations sharing one rule), + # and overlapping run selections re-verify already-covered rules. + remaining_rules.discard(k.rule) + if not remaining_rules: + return True + return False + +def _merge_prover_history(left: list[ProverHistoryItem], right: list[ProverHistoryItem]) -> list[ProverHistoryItem]: + to_ret = left.copy() + to_ret.extend(right) + return to_ret class ProverStateExtra(TypedDict): rule_skips: Annotated[dict[str, str], _merge_rule_skips] @@ -105,6 +259,9 @@ class ProverStateExtra(TypedDict): # Basename the spec is materialized/persisted under (e.g. "autospec_"). # NotRequired so other ProverStateExtra injectors (e.g. config_edit) needn't set it. spec_stem: NotRequired[str] + prover_history: Annotated[list[ProverHistoryItem], _merge_prover_history] + reminders_channel: list[str] + # The author's working copy of the source under verification; verify_spec runs # against its materialization when non-empty (see ProjectDirectory). Absent/empty # outside the editing-enabled pipeline. No merge op intentionally: the vfs is @@ -228,8 +385,14 @@ class VerifySpecSchema(BaseModel): rules: list[str] | None = Field( default=None, - description="Specific rules to verify. If None, verifies all rules." + description="Specific rules to verify. If None, verifies all rules. Mutually exclusive with the `exclude_rules` argument" ) + + exclude_rules: list[str] | None = Field( + default=None, + description="Specific rules to SKIP verifying. If none validates all rules. Mutually exclusive with `rules` argument" + ) + state: Annotated[StateWithSkips, InjectedState] @@ -255,15 +418,8 @@ def tmp_spec( def _prover_sem(cloud: bool) -> AsyncContextManager[None]: if not cloud: return asyncio.Semaphore(1) - - class ToRet(): - async def __aenter__(self): - return - - async def __aexit__(self, exc_type, exc, tb): - return - - return ToRet() + else: + return nullcontext() type ProjectDirectory = Callable[[dict[str, str]], AsyncContextManager[str]] @@ -303,6 +459,42 @@ async def provide(vfs: dict[str, str]) -> AsyncIterator[str]: await asyncio.to_thread(stack.close) return provide +@contextmanager +def setup_prover_config_in( + *, + working_dir: str, + config: dict, + spec_contents: str, + spec_stem: str | None = None, + main_contract: str, + rule: list[str] | None, + exclude_rule: list[str] | None, + conf_dir: Path = CERTORA_DIR, + **config_extra +): + with tmp_spec( + root=working_dir, + content=spec_contents, + name=spec_stem + ) as generated_path: + config = prover_config_overlay( + config, main_contract=main_contract, verify_target=f"{main_contract}:{generated_path}" + ) + config.update(config_extra) + if rule is not None: + config["rule"] = rule + if exclude_rule is not None: + config["exclude_rule"] = exclude_rule + with temp_certora_file( + root=working_dir, + content=json.dumps(config, indent=2), + ext="conf", + name=spec_stem, + prefix="verify", + dest_dir=conf_dir, + ) as conf_path: + yield (conf_path, config) + def get_prover_tool( llm: LLM, main_contract: str, @@ -326,70 +518,166 @@ def get_prover_tool( async def verify_spec( tool_call_id: Annotated[str, InjectedToolCallId], state: Annotated[StateWithSkips, InjectedState], - rules: list[str] | None = None + rules: list[str] | None = None, + exclude_rules: list[str] | None = None ) -> str | Command: + last_msg = state["messages"][-1] + if isinstance(last_msg, AIMessage) and any( + i["id"] != tool_call_id for i in last_msg.tool_calls + ): + return "Cannot call the verify_spec tool in parallel with other tool calls. verify_spec must be the only tool you call in a turn" + + if rules is not None and exclude_rules is not None: + return "Cannot invoke the prover with both `rules` and `exclude_rules` set to non-none" + spec = state["curr_spec"] if spec is None: return "Specification not yet put on VFS" + + spec_hash = string_hash( + spec + ) + + if (last_run := last_prover_run(state["prover_history"])) is not None: + if any(i == "TIMEOUT" for (_,i) in last_run["prover_results"]) and last_run["spec_digest"] == spec_hash: + return "Refusing to re-run prover on identical spec with a known TIMEOUT result; timeouts are not transient " \ + "errors and will not go away by re-running the tool." + conf = state["config"] # With a seeded stem, name the spec/conf after it (so on-disk names match the # dump) under a lock; else fall back to unique uid names (no lock needed). spec_stem = state.get("spec_stem") + summary = get_run_summary() + component = (spec_stem or main_contract).removeprefix("autospec_") + iteration = len(state["prover_history"]) + 1 + conf_dir = (CERTORA_DIR / "confs") if spec_stem is not None else CERTORA_DIR lock = spec_locks.setdefault(spec_stem, asyncio.Lock()) if spec_stem is not None else nullcontext() + prover_msg = f"{component} iteration number {iteration}" + + summary = get_run_summary() + + component = (spec_stem or main_contract).removeprefix("autospec_") + iteration = len(state["prover_history"]) + 1 + prover_msg = f"{component} iteration number {iteration}" + async def run_in(run_root: str) -> str | Command: - with tmp_spec(root=run_root, content=spec, name=spec_stem) as generated: - config = prover_config_overlay( - conf, main_contract=main_contract, verify_target=f"{main_contract}:{generated}" + with setup_prover_config_in( + working_dir=run_root, + main_contract=main_contract, + spec_stem=spec_stem, + spec_contents=spec, + conf_dir=conf_dir, + config=conf, + rule=None, + exclude_rule=None, + msg="" + ) as (config_path, _ignored): + all_rules = await declared_rules_list( + folder=Path(run_root), + args=[config_path] ) + with setup_prover_config_in( + working_dir=run_root, + main_contract=main_contract, + spec_stem=spec_stem, + spec_contents=spec, + conf_dir=conf_dir, + config=conf, + rule=rules, + exclude_rule=exclude_rules, + msg=prover_msg + ) as (config_path, config): + async with sem: + result = await run_prover( + Path(run_root), + [config_path], + tool_call_id, + prover_opts, + _SpecCallbacks(get_stream_writer(), tool_call_id, summary, config, + analysis_store=analysis_store), + DefaultCexHandler(llm, state, summarization_threshold=10) + ) - if rules: - config["rule"] = rules - - summary = get_run_summary() - - component = (spec_stem or main_contract).removeprefix("autospec_") - iteration = summary.prover_total_calls + 1 - config["msg"] = f"{component} iteration number {iteration}" - - with temp_certora_file( - root=run_root, - content=json.dumps(config, indent=2), - ext="conf", - name=spec_stem, - prefix="verify", - dest_dir=conf_dir, - ) as config_path: - async with sem: - result = await run_prover( - Path(run_root), - [config_path], - tool_call_id, - prover_opts, - _SpecCallbacks(get_stream_writer(), tool_call_id, summary, config, - analysis_store=analysis_store), - DefaultCexHandler(llm, state, summarization_threshold=10) - ) - - if isinstance(result, str): - return result - all_verified = True - for (r, stat) in result.rule_status.items(): - if r in state["rule_skips"]: - continue - if not stat: - all_verified = False - break - if rules is None and all_verified: - return tool_state_update( - tool_call_id=tool_call_id, content=result.result_str, - prover_link=result.link, - validations=stamper(state, state["version_history"]), + if isinstance(result, str): + return result + + stuck_rules = { + k: v for (k,v) in result.raw_rule_status.items() if v in ("TIMEOUT", "ERROR", "SANITY_FAILED") and k.rule not in state["rule_skips"] + } + + known_tc_ids = { + l["id"] + for msg in state["messages"] if isinstance(msg, AIMessage) + for l in msg.tool_calls if l["name"] == "verify_spec" + } + + to_warn, seen_post_compaction_history = stuck_rule_warnings( + stuck_rules, state["prover_history"], known_tc_ids + ) + + curr_state_digest = spec_digest( + spec, state["skipped"], state["version_history"] + ) + + prover_results : list[tuple[RulePath, StatusCodes]] = [(k, v) for (k,v) in result.raw_rule_status.items()] + + all_verified = _is_completion_history( + l=state["prover_history"], + curr_digest=curr_state_digest, + expected_to_fail=set(state["rule_skips"].keys()), + curr_status=prover_results, + all_rules=all_rules + ) + + prover_update : list[ProverHistoryItem] = [ + ProverRunLog( + tool_call_id=tool_call_id, + prover_results=[(k, v) for (k,v) in result.raw_rule_status.items()], + rules={"sort": "exclude", "selector": exclude_rules } if exclude_rules is not None else \ + {"sort": "include", "selector": rules} if rules is not None else None, + spec_digest=spec_hash, + sort="run", + declared_rules=all_rules, + state_digest=curr_state_digest + ) + ] + nag_channel = { + + } + if len(to_warn) > 0: + prover_update.append(NagMarker( + sort="nag", + nagged_rules=list(to_warn) + )) + nag_channel["reminders_channel"] = [ + "The following rule(s) have had identical failures on the last 3 runs of the prover:", + *(f"- {it.pprint()}" for it in to_warn), + "You may need to significantly change your approach, or skip the property if this is a persistent issue (you may need to use rebuttals to communicate" + " these failures to the feedback judge)." + ] + if seen_post_compaction_history: + nag_channel["reminders_channel"].append( + "(NB: Some of these prover calls happened before your most recent task history summarization)" ) + if all_verified: + nag_channel.setdefault("reminders_channel", []).append( + "You have successfully verified over your prior prover run(s) that all rules verify. This task is completed." + ) + # Completing the coverage stamps, however the completing run was scoped: + # every declared rule was verified against exactly this authoring state + # (the state_digest match), so a piecemeal completion is as good as a + # full-run one. return tool_state_update( - tool_call_id=tool_call_id, content=result.result_str, prover_link=result.link + tool_call_id=tool_call_id, content=result.result_str, + prover_link=result.link, validations=stamper(state, state["version_history"]), + prover_history=prover_update, **nag_channel ) + return tool_state_update( + tool_call_id=tool_call_id, content=result.result_str, prover_link=result.link, + prover_history=prover_update, **nag_channel + ) # The author's working copy decides where this run executes (in-situ for # an empty VFS, a temp materialization otherwise); the same-stem lock diff --git a/composer/spec/source/report/build.py b/composer/spec/source/report/build.py index e9cecbc2..e9eda26c 100644 --- a/composer/spec/source/report/build.py +++ b/composer/spec/source/report/build.py @@ -12,16 +12,18 @@ from langchain_core.language_models.chat_models import BaseChatModel +from composer.spec.types import Curtailed from composer.spec.source.report.collect import ( EvidenceFetcher, ReportableResult, ReportComponentInput, VerdictFetcher, collect, ) from composer.spec.source.report.coverage import ValidationError, validate from composer.spec.source.report.findings import build_findings from composer.spec.source.report.grouping import ( - build_fallback_grouping, build_groups, call_grouping_llm, + build_fallback_grouping, build_groups, call_grouping_llm, PropertyGroup ) from composer.spec.source.report.schema import ( - AutoProverReport, Finding, Outcome, PropertyKey, ReportBackend, RuleRef, SourceEditRecord + AutoProverReport, Finding, Outcome, PropertyKey, ReportBackend, RuleRef, SourceEditRecord, + VerificationArtifactRecord, ) _log = logging.getLogger(__name__) @@ -44,6 +46,7 @@ async def build_report[R: ReportableResult]( llm: BaseChatModel, fetch_verdicts: VerdictFetcher[R], source_edits: list[SourceEditRecord] | None = None, + verification_artifacts: list[VerificationArtifactRecord] | None = None, findings_llm: BaseChatModel | None = None, fetch_evidence: EvidenceFetcher | None = None, ) -> AutoProverReport: @@ -53,7 +56,7 @@ async def build_report[R: ReportableResult]( audit-issue `Finding`s (best-effort; a synthesis failure yields no findings rather than failing the report). ``fetch_evidence`` supplies each violation's captured counterexample analysis; it is optional.""" - properties, rules, skipped, gave_up, dropped = await collect( + properties, rules, skipped, gave_up, curtailed, dropped = await collect( components, fetch_verdicts=fetch_verdicts ) rule_outcomes: dict[RuleRef, Outcome] = {r.ref: r.outcome for r in rules} @@ -63,36 +66,53 @@ async def build_report[R: ReportableResult]( # always has a high-level section: (a) the LLM call raises, (b) validation rejects a # structurally-invalid grouping, (c) the grouping is valid but covers no properties. The # fallback bucket holds every property exactly once, so the re-validate below cannot raise. + # With no formalized properties at all (everything gave up or was budget-curtailed), there is + # nothing to group and no reason to spend the LLM call. fallback_reason: str | None = None - try: - grouping = await call_grouping_llm( - llm=llm, contract_name=contract_name, properties=properties, - ) - groups = build_groups(grouping.groups, props_by_key, rule_outcomes) + if not properties: + groups: list[PropertyGroup] = [] coverage = validate( - properties=properties, rules=rules, groups=groups, - skipped=skipped, gave_up=gave_up, dropped_orphan_rules=dropped, - ) - grouped: set[PropertyKey] = {k for g in groups for k in g.members} - if properties and not grouped: - raise ValidationError("grouping produced no high-level properties") - except Exception as e: # noqa: BLE001 — any LLM/transport/validation error degrades - if RERAISE_REPORT_FAILURES: - raise - fallback_reason = ( - f"validation rejected the grouping: {e}" if isinstance(e, ValidationError) - else f"grouping failed: {e}" + properties=properties, rules=rules, groups=groups, skipped=skipped, + gave_up=gave_up, curtailed=curtailed, dropped_orphan_rules=dropped, ) - _log.warning("report: %s; applying fallback grouping", fallback_reason) - groups = build_groups( - build_fallback_grouping(properties).groups, props_by_key, rule_outcomes - ) - coverage = validate( - properties=properties, rules=rules, groups=groups, - skipped=skipped, gave_up=gave_up, dropped_orphan_rules=dropped, - ) - coverage.warnings = ["FALLBACK GROUPING APPLIED"] + coverage.warnings + else: + try: + grouping = await call_grouping_llm( + llm=llm, contract_name=contract_name, properties=properties, + ) + groups = build_groups(grouping.groups, props_by_key, rule_outcomes) + coverage = validate( + properties=properties, rules=rules, groups=groups, skipped=skipped, + gave_up=gave_up, curtailed=curtailed, dropped_orphan_rules=dropped, + ) + grouped: set[PropertyKey] = {k for g in groups for k in g.members} + if not grouped: + raise ValidationError("grouping produced no high-level properties") + except Exception as e: # noqa: BLE001 — any LLM/transport/validation error degrades + if RERAISE_REPORT_FAILURES: + raise + fallback_reason = ( + f"validation rejected the grouping: {e}" if isinstance(e, ValidationError) + else f"grouping failed: {e}" + ) + _log.warning("report: %s; applying fallback grouping", fallback_reason) + groups = build_groups( + build_fallback_grouping(properties).groups, props_by_key, rule_outcomes + ) + coverage = validate( + properties=properties, rules=rules, groups=groups, skipped=skipped, + gave_up=gave_up, curtailed=curtailed, dropped_orphan_rules=dropped, + ) + coverage.warnings = ["FALLBACK GROUPING APPLIED"] + coverage.warnings + # Curtailed results are deliberately absent: a run link on a partial encoding proves nothing + # about it, so it lives only in the component's appendix record. + prover_links = { + c.name: c.formalized.run_link + for c in components + if c.formalized is not None and not isinstance(c.formalized, Curtailed) + and c.formalized.run_link + } # Violated rules -> findings. Its own guard: findings synthesis must never fail the report # (the whole phase is also best-effort in the caller, but this keeps a working report even when # only findings break). @@ -112,14 +132,15 @@ async def build_report[R: ReportableResult]( backend=backend, contract_name=contract_name, run_timestamp_utc=datetime.now(timezone.utc).isoformat(), - prover_links={c.name: c.formalized.run_link for c in components - if c.formalized and c.formalized.run_link}, + prover_links=prover_links, properties=properties, rules=rules, groups=groups, skipped=skipped, gave_up_components=gave_up, + curtailed_components=curtailed, source_edits=source_edits or [], + verification_artifacts=verification_artifacts or [], coverage=coverage, findings=findings, ) diff --git a/composer/spec/source/report/collect.py b/composer/spec/source/report/collect.py index e0d9e210..aa23cf17 100644 --- a/composer/spec/source/report/collect.py +++ b/composer/spec/source/report/collect.py @@ -2,21 +2,25 @@ For each component (and the structural invariants) the report phase hands us the inferred properties, the generation result (a `ReportableResult`: its skip list + property->unit mapping; -``None`` if the component gave up or crashed), and a per-component run link. We split the -properties into the ones a rule formalizes (`FormalizedProperty`) and the formalization gaps -(`SkippedClaim` / `GaveUpComponent`), and fetch per-unit `Outcome`s via a backend-supplied -`VerdictFetcher`. No on-disk dumps are read — the data is already in memory. +a `Curtailed` wrapper when the budget cut the generation short; ``None`` if the component gave up +or crashed), and a per-component run link. We split the properties into the ones a rule formalizes +(`FormalizedProperty`) and the formalization gaps (`SkippedClaim` / `GaveUpComponent`), route +budget-curtailed components into `CurtailedComponent` appendix records (their encodings and +verdicts are unreliable, so they are neither verdict-fetched nor grouped), and fetch per-unit +`Outcome`s via a backend-supplied `VerdictFetcher`. No on-disk dumps are read — the data is +already in memory. """ import asyncio import logging from dataclasses import dataclass -from typing import Awaitable, Callable, Protocol +from pathlib import Path +from typing import Protocol -from composer.spec.cvl_generation import SkippedProperty -from composer.spec.types import PropertyFormulation +from composer.authoring.state import SkippedProperty +from composer.spec.types import Curtailed, PropertyFormulation from composer.spec.source.report.schema import ( - ComponentName, FormalizedProperty, GaveUpComponent, Outcome, PropertyTitle, RuleName, RuleRef, - RuleVerdict, SkippedClaim, + ComponentName, CurtailedComponent, CurtailedSkip, DraftedProperty, FormalizedProperty, + GaveUpComponent, Outcome, PropertyTitle, RuleName, RuleRef, RuleVerdict, SkippedClaim, ) _log = logging.getLogger(__name__) @@ -25,12 +29,12 @@ class ReportableResult(Protocol): """The backend-agnostic view the report needs of a successful generation result. Both `GeneratedCVL` and `GeneratedFoundryTest` satisfy it: ``skipped`` are the properties the author - declined, and ``property_units()`` is the property->formalizing-units adapter (CVL rules / + declined, and ``property_checks()`` is the property->checks adapter (CVL rules / foundry tests — the underlying field names differ, hence the method rather than structural matching).""" skipped: list[SkippedProperty] - def property_units(self) -> list[tuple[PropertyTitle, list[RuleName]]]: ... + def property_checks(self) -> list[tuple[PropertyTitle, list[RuleName]]]: ... @property def output_link(self) -> str | None: @@ -40,13 +44,15 @@ def output_link(self) -> str | None: class Formalized[R: ReportableResult](Protocol): - """The report's view of a successful generation: the result, the basename of the file its units - live in (``autospec_.spec`` / ``invariants.spec`` / a ``.t.sol``) — the unit-identity - fallback when a verdict carries no source location — and the verification-run link (``None`` for - backends with no run service).""" + """The report's view of a generation result persisted to disk: the result, the project-relative + path it was written to, the basename of the file its units live in (``autospec_.spec`` / + ``invariants.spec`` / a ``.t.sol``) — the unit-identity fallback when a verdict carries no + source location — and the verification-run link (``None`` for backends with no run service).""" @property def result(self) -> R: ... @property + def deliverable(self) -> Path: ... + @property def unit_file(self) -> str: ... @property def run_link(self) -> str | None: ... @@ -55,11 +61,13 @@ def run_link(self) -> str | None: ... @dataclass(frozen=True) class ReportComponentInput[R: ReportableResult]: """One unit to collect: a component or the structural invariants. ``formalized`` carries the - generation result and its unit file / run link, or is ``None`` when the component gave up or - crashed — in which case no units were formalized, no file was written, and there is no run.""" + generation result and its unit file / run link; a `Curtailed` wrapper when the budget cut the + generation short (its ``partial`` is the quarantined encoding, or ``None`` if nothing was + published); or ``None`` when the component gave up or crashed — in which case no units were + formalized, no file was written, and there is no run.""" name: ComponentName props: list[PropertyFormulation] - formalized: Formalized[R] | None + formalized: "Formalized[R] | Curtailed[Formalized[R]] | None" @dataclass(frozen=True) @@ -69,10 +77,14 @@ class Verdict: line: int | None = None duration_seconds: float | None = None unit_file: str | None = None + #: Human-readable explanation of a non-GOOD outcome (a counterexample / assertion message for + #: a BAD, error text for an ERROR). Provenance/diagnostics only; ``None`` when the backend + #: gives no detail (the prover/foundry fetchers don't). + message: str | None = None def merge(self, other: "Verdict | None") -> "Verdict": """Combine two results for one unit within a run: higher-priority outcome wins, - line/duration/unit_file kept from whichever side has them.""" + line/duration/unit_file/message kept from whichever side has them.""" if other is None: return self hi, lo = ( @@ -85,6 +97,7 @@ def merge(self, other: "Verdict | None") -> "Verdict": hi.line if hi.line is not None else lo.line, hi.duration_seconds if hi.duration_seconds is not None else lo.duration_seconds, hi.unit_file or lo.unit_file, + hi.message or lo.message, ) @@ -94,13 +107,51 @@ def merge(self, other: "Verdict | None") -> "Verdict": } class VerdictFetcher[R: ReportableResult](Protocol): - """Backend hook: given one collected input, return its units' verdicts keyed by unit name. The - prover impl calls ProverOutputUtility off-thread; the foundry impl reads the result's ran/expected - tests. A component with no result (gave up) yields ``{}``.""" - async def __call__(self, input: ReportComponentInput[R], /) -> dict[RuleName, Verdict]: + """Backend hook: given one delivered result, return its units' verdicts keyed by unit name. The + prover impl calls ProverOutputUtility off-thread; the foundry impl reads the result's + ran/expected tests. Only invoked for genuinely formalized inputs — gave-up and curtailed + components have no verdicts to fetch.""" + async def __call__(self, formalized: Formalized[R], /) -> dict[RuleName, Verdict]: ... +def _curtailed_component[R: ReportableResult]( + name: ComponentName, + props: list[PropertyFormulation], + c: Curtailed["Formalized[R]"], +) -> CurtailedComponent: + """Partition a curtailed component's inferred properties by what its partial result (if any) + says happened to them: claimed-encoded (``drafted``, unverified), explicitly ``skipped``, or + ``unattempted``. With no published partial, everything is unattempted.""" + if c.partial is None: + return CurtailedComponent( + component=name, detail=c.detail, unattempted=list(props), + ) + res = c.partial.result + skip_reasons = {s.property_title: s.reason for s in res.skipped} + mapping = dict(res.property_checks()) + drafted: list[DraftedProperty] = [] + skipped: list[CurtailedSkip] = [] + unattempted: list[PropertyFormulation] = [] + for p in props: + if p.title in skip_reasons: + skipped.append(CurtailedSkip(reason=skip_reasons[p.title], **p.model_dump())) + elif p.title in mapping: + drafted.append(DraftedProperty( + units=[u for u in mapping[p.title] if u.strip()], **p.model_dump(), + )) + else: + unattempted.append(p) + return CurtailedComponent( + component=name, + artifact=str(c.partial.deliverable), + run_link=c.partial.run_link, + detail=c.detail, + drafted=drafted, + skipped=skipped, + unattempted=unattempted, + ) + @dataclass(frozen=True) class RuleEvidence: """One failing instance of a violated rule: the backend's root-cause explanation and a concrete @@ -123,20 +174,31 @@ async def collect[R: ReportableResult]( inputs: list[ReportComponentInput[R]], *, fetch_verdicts: VerdictFetcher[R], -) -> tuple[list[FormalizedProperty], list[RuleVerdict], list[SkippedClaim], list[GaveUpComponent], int]: +) -> tuple[ + list[FormalizedProperty], list[RuleVerdict], list[SkippedClaim], list[GaveUpComponent], + list[CurtailedComponent], int, +]: """Assemble the report inputs. - Returns ``(formalized_properties, rules, skipped, gave_up_components, dropped_orphan_count)``. - Rules are identified by ``(unit_file, name)``: a single definition seen through several runs - (e.g. a structural invariant imported into a component spec) collapses to one entry. Orphan - units — reported by the backend but referenced by no property — are dropped and counted. - Verdicts are fetched concurrently via the backend `fetch_verdicts` hook. + Returns ``(formalized_properties, rules, skipped, gave_up_components, curtailed_components, + dropped_orphan_count)``. Rules are identified by ``(unit_file, name)``: a single definition + seen through several runs (e.g. a structural invariant imported into a component spec) + collapses to one entry. Orphan units — reported by the backend but referenced by no property — + are dropped and counted. Verdicts are fetched concurrently via the backend `fetch_verdicts` + hook, for delivered inputs only: a curtailed component's verification state is unreliable by + construction, so nothing is fetched for it. """ - verdict_maps = await asyncio.gather(*[fetch_verdicts(inp) for inp in inputs]) + async def _verdicts(inp: ReportComponentInput[R]) -> dict[RuleName, Verdict]: + if inp.formalized is None or isinstance(inp.formalized, Curtailed): + return {} + return await fetch_verdicts(inp.formalized) + + verdict_maps = await asyncio.gather(*[_verdicts(inp) for inp in inputs]) properties: list[FormalizedProperty] = [] skipped: list[SkippedClaim] = [] gave_up: list[GaveUpComponent] = [] + curtailed: list[CurtailedComponent] = [] rules_by_key: dict[RuleRef, RuleVerdict] = {} referenced: set[RuleRef] = set() @@ -145,13 +207,16 @@ async def collect[R: ReportableResult]( # Gave up or crashed: the whole component is a formalization gap. gave_up.append(GaveUpComponent(component=inp.name, properties=inp.props)) continue + if isinstance(inp.formalized, Curtailed): + curtailed.append(_curtailed_component(inp.name, inp.props, inp.formalized)) + continue res = inp.formalized.result unit_file = inp.formalized.unit_file run_link = inp.formalized.run_link skip_reasons = {s.property_title: s.reason for s in res.skipped} - mapping = dict(res.property_units()) + mapping = dict(res.property_checks()) - def _ref(unit_name: str) -> RuleRef: + def _ref(unit_name: RuleName) -> RuleRef: v = verdicts.get(unit_name) return ((v.unit_file if v and v.unit_file else unit_file), unit_name) @@ -180,7 +245,7 @@ def _ref(unit_name: str) -> RuleRef: if key not in rules_by_key: rules_by_key[key] = RuleVerdict( name=unit_name, spec_file=key[0], outcome=v.outcome, line=v.line, - duration_seconds=v.duration_seconds, prover_link=run_link, + duration_seconds=v.duration_seconds, prover_link=run_link, message=v.message, ) # A referenced unit with no verdict still needs an (UNKNOWN) entry to render. @@ -193,4 +258,4 @@ def _ref(unit_name: str) -> RuleRef: key=lambda r: r.ref, ) dropped_orphans = sum(1 for key in rules_by_key if key not in referenced) - return properties, rules, skipped, gave_up, dropped_orphans + return properties, rules, skipped, gave_up, curtailed, dropped_orphans diff --git a/composer/spec/source/report/coverage.py b/composer/spec/source/report/coverage.py index 567a6bf1..10cd960f 100644 --- a/composer/spec/source/report/coverage.py +++ b/composer/spec/source/report/coverage.py @@ -9,8 +9,8 @@ from collections import Counter, defaultdict from composer.spec.source.report.schema import ( - CoverageReport, FormalizedProperty, GaveUpComponent, PropertyGroup, PropertyKey, - RuleRef, RuleVerdict, SkippedClaim, + CoverageReport, CurtailedComponent, FormalizedProperty, GaveUpComponent, PropertyGroup, + PropertyKey, RuleRef, RuleVerdict, SkippedClaim, ) @@ -25,6 +25,7 @@ def validate( groups: list[PropertyGroup], skipped: list[SkippedClaim], gave_up: list[GaveUpComponent], + curtailed: list[CurtailedComponent], dropped_orphan_rules: int, ) -> CoverageReport: """Cross-check the grouping against the property set; produce a `CoverageReport`. @@ -61,6 +62,13 @@ def validate( groups_of_rule[ref].add(g.slug) spanning = sorted({ref[1] for ref, slugs in groups_of_rule.items() if len(slugs) > 1}) + warnings: list[str] = [] + if curtailed: + warnings.append( + f"{len(curtailed)} component(s) were cut short by the run budget; their properties " + "are excluded from the groupings above (see the budget appendix)." + ) + sizes = [len(g.members) for g in groups] return CoverageReport( total_properties=len(properties), @@ -73,5 +81,7 @@ def validate( rules_spanning_multiple_groups=spanning, skipped_count=len(skipped), gave_up_component_count=len(gave_up), + curtailed_component_count=len(curtailed), dropped_orphan_rules=dropped_orphan_rules, + warnings=warnings, ) diff --git a/composer/spec/source/report/render.py b/composer/spec/source/report/render.py index b7d22980..faba68a6 100644 --- a/composer/spec/source/report/render.py +++ b/composer/spec/source/report/render.py @@ -3,7 +3,9 @@ Single self-contained page (inline CSS, no external assets): a header with outcome counts, one section per high-level `PropertyGroup` (status badge + description + a rule table whose per-rule descriptions are the in-group property claims that pull each rule in), a formalization-gaps section -(declined properties + components that gave up), and a coverage footer. The HTML is built by +(declined properties + components that gave up), an appendix for components the run budget cut +short (counts-first summary per component, collapsed per-property breakdown), and a coverage +footer. The HTML is built by ``autoprove_report.html.j2``; this module only assembles the render context — no markup here. The template's parameters are typed by `ReportTemplateParams` and rendered through the `TypedTemplate` infra, so a context/template drift is a type error. @@ -22,13 +24,14 @@ from collections import Counter from pathlib import Path from typing import TypedDict +from collections.abc import Sequence from composer.spec.gen_types import TypedTemplate from composer.templates.loader import load_jinja_template from composer.spec.source.report.schema import ( - AutoProverReport, CoverageReport, Finding, FormalizedProperty, GaveUpComponent, GroupStatus, Outcome, - PropertyGroup, PropertyKey, ReportBackend, RuleRef, RuleVerdict, SkippedClaim, - SourceEditRecord, + AutoProverReport, ComponentName, CoverageReport, CurtailedComponent, Finding, + FormalizedProperty, GaveUpComponent, GroupStatus, Outcome, PropertyGroup, PropertyKey, + ReportBackend, RuleRef, RuleVerdict, SkippedClaim, SourceEditRecord, ) @@ -66,6 +69,13 @@ Outcome.GOOD: "Successful test", Outcome.BAD: "Failing test", Outcome.ERROR: "Error", Outcome.TIMEOUT: "Timeout", Outcome.UNKNOWN: "Unknown", }, + # The analysis-only backend never produces a verdict, so UNKNOWN is the label that matters: + # "Unverified" states why the row is empty, where "Unknown" would read as a failed attempt. + # The rest are neutral words, present because the table has to be total. + "none": { + Outcome.GOOD: "Passed", Outcome.BAD: "Failed", Outcome.ERROR: "Error", + Outcome.TIMEOUT: "Timeout", Outcome.UNKNOWN: "Unverified", + }, } _GROUP_LABELS: dict[ReportBackend, dict[GroupStatus, str]] = { "prover": { @@ -76,6 +86,10 @@ GroupStatus.GOOD: "All tests passing", GroupStatus.BAD: "Has failing test", GroupStatus.PARTIAL: "Partial", GroupStatus.UNKNOWN: "No results", }, + "none": { + GroupStatus.GOOD: "All passing", GroupStatus.BAD: "Has failure", + GroupStatus.PARTIAL: "Partial", GroupStatus.UNKNOWN: "Unverified", + }, } @@ -98,6 +112,10 @@ class ReportTerms(TypedDict): title="Foundry test report", unit_singular="test", unit_plural="tests", unit_cap="Test", outcomes_label="Test outcomes", ), + "none": ReportTerms( + title="Property report", unit_singular="property", unit_plural="properties", + unit_cap="Property", outcomes_label="Property outcomes", + ), } # Chip display order for the header outcome counts. @@ -132,6 +150,9 @@ class RowView(TypedDict): line: int | None link: LinkView descriptions: list[str] + #: Backend diagnostic for a non-GOOD row (e.g. the fuzzer's counterexample / failed-assertion + #: message). ``None`` when the backend supplied none. + message: str | None class GroupView(TypedDict): @@ -163,6 +184,28 @@ class FindingView(TypedDict): spec_file: str | None +class CurtailedRowView(TypedDict): + """One property row in a curtailed component's per-property breakdown. ``units`` (drafted + rows) and ``note`` (skip reason) feed the Notes cell; both empty renders an em-dash.""" + description: str + sort: str + label: str + kind: str + units: Sequence[str] + note: str | None + + +class CurtailedView(TypedDict): + component: str + status_label: str + status_kind: str + summary: str + artifact: str | None + link: LinkView + detail: str | None + rows: list[CurtailedRowView] + + class ReportTemplateParams(TypedDict): """The full, typed context of ``autoprove_report.html.j2``.""" contract_name: str @@ -177,12 +220,42 @@ class ReportTemplateParams(TypedDict): groups: list[GroupView] skipped: list[SkippedClaim] gave_up: list[GaveUpComponent] + curtailed: list[CurtailedView] source_edits: list[SourceEditRecord] _REPORT_TEMPLATE = TypedTemplate[ReportTemplateParams]("autoprove_report.html.j2") +def outcome_label(backend: ReportBackend, outcome: Outcome) -> str: + """The human word an auditor reads for an ``Outcome`` under a backend (e.g. a ``prover`` + ``GOOD`` → "Verified", a ``foundry`` ``GOOD`` → "Successful test"). + + The report's HTML render is the primary consumer, but the console/TUI verdict + rollups reuse this so the same run reads with one vocabulary everywhere — this is + the single place the per-backend wording lives.""" + return _OUTCOME_LABELS[backend][outcome] + + +#: One glyph per outcome, for the places a verdict has to scan at a glance rather than read: the +#: console rollup's per-unit listing and the TUI's notice callouts. Backend-independent, unlike the +#: labels — a ✓ means the same thing whichever prover produced it. +_OUTCOME_GLYPHS: dict[Outcome, str] = { + Outcome.GOOD: "✓", + Outcome.BAD: "✗", + Outcome.TIMEOUT: "⧖", + Outcome.ERROR: "!", + Outcome.UNKNOWN: "?", +} + + +def outcome_glyph(outcome: Outcome) -> str: + """The at-a-glance mark for an ``Outcome``. Lives beside :func:`outcome_label` because it answers + the same question — how this outcome reads to a human — and every surface that shows a glyph + must show the same one.""" + return _OUTCOME_GLYPHS[outcome] + + def _is_url(link: str) -> bool: return link.startswith("http://") or link.startswith("https://") @@ -220,7 +293,7 @@ def _group_view( rules_by_ref: dict[RuleRef, RuleVerdict], unit_labels: dict[Outcome, str], group_labels: dict[GroupStatus, str], - edited_components: set[str], + edited_components: set[ComponentName], ) -> GroupView: """Invert the group's members into rule rows: each rule the group's properties formalize, labelled with the descriptions of the in-group properties that pull it in (the edge labels). The same rule @@ -249,6 +322,7 @@ def _group_view( "line": rule.line if rule else None, "link": _link_view(rule.prover_link if rule else None), "descriptions": descriptions[ref], + "message": rule.message if rule else None, }) return { "slug": group.slug, @@ -283,6 +357,50 @@ def _finding_view(f: Finding) -> FindingView: } +def _plural(n: int, singular: str, plural: str) -> str: + return f"{n} {singular if n == 1 else plural}" + + +def _curtailed_view(c: CurtailedComponent) -> CurtailedView: + """One appendix card: a counts-first summary sentence, then a per-property breakdown row per + inferred property (disposition badge + the declared units / skip reason as notes).""" + total = len(c.drafted) + len(c.skipped) + len(c.unattempted) + parts: list[str] = [] + if c.drafted: + parts.append(f"{len(c.drafted)} drafted but never verified") + if c.skipped: + parts.append(f"{len(c.skipped)} skipped") + if c.unattempted: + parts.append(f"{len(c.unattempted)} never attempted") + summary = f"Of {_plural(total, 'inferred property', 'inferred properties')}: {', '.join(parts)}." + + rows: list[CurtailedRowView] = [] + for d in c.drafted: + rows.append({"description": d.description, "sort": d.sort, + "label": "Drafted — unverified", "kind": "warn", + "units": d.units, "note": None}) + for s in c.skipped: + rows.append({"description": s.description, "sort": s.sort, + "label": "Skipped", "kind": "muted", + "units": [], "note": s.reason}) + for p in c.unattempted: + rows.append({"description": p.description, "sort": p.sort, + "label": "Not attempted", "kind": "muted", + "units": [], "note": None}) + + published = c.artifact is not None + return { + "component": c.component, + "status_label": "partial draft published" if published else "nothing published", + "status_kind": "warn" if published else "bad", + "summary": summary, + "artifact": c.artifact, + "link": _link_view(c.run_link), + "detail": c.detail, + "rows": rows, + } + + def _build_context(report: AutoProverReport) -> ReportTemplateParams: props_by_key = {p.key: p for p in report.properties} rules_by_ref = {r.ref: r for r in report.rules} @@ -309,6 +427,7 @@ def _build_context(report: AutoProverReport) -> ReportTemplateParams: ], "skipped": report.skipped, "gave_up": report.gave_up_components, + "curtailed": [_curtailed_view(c) for c in report.curtailed_components], "source_edits": report.source_edits, } diff --git a/composer/spec/source/report/schema.py b/composer/spec/source/report/schema.py index 592a0752..a897fd09 100644 --- a/composer/spec/source/report/schema.py +++ b/composer/spec/source/report/schema.py @@ -15,17 +15,7 @@ from pydantic import BaseModel, Field -from composer.spec.types import PropertyFormulation - -type RuleName = str -"""A CVL rule/invariant identifier as it appears in the prover report and in a component's -``property_rules`` mapping.""" - -type ComponentName = str -"""Human name of an AIComposer component (e.g. "Increment"), or "Structural Invariants".""" - -type PropertyTitle = str -"""A property's unique snake_case title — the key in a component's ``property_rules`` mapping.""" +from composer.spec.types import ComponentName, PropertyFormulation, PropertyTitle, RuleName type RuleRef = tuple[str, RuleName] """A rule's identity: ``(spec_file, name)``. A name is only unique within a spec, so the defining @@ -55,6 +45,19 @@ class Outcome(str, Enum): TIMEOUT = "TIMEOUT" UNKNOWN = "UNKNOWN" + @classmethod + def parse(cls, raw: str) -> "Outcome | None": + """``raw`` as an outcome, or ``None`` when it names none. + + For values arriving from outside this repo — a backend wheel's JSON, a ``report.json`` read + cold — where an unrecognized label is version skew rather than a bug, and the caller decides + what to do about it (record UNKNOWN, drop a glyph). For a value we produced ourselves, + ``Outcome(raw)`` and its ValueError remain the right thing.""" + try: + return cls(raw) + except ValueError: + return None + class GroupStatus(str, Enum): """Aggregated outcome for a `PropertyGroup`, rolled up from the `Outcome` of the rules its @@ -84,6 +87,11 @@ class RuleVerdict(BaseModel): line: int | None = None duration_seconds: float | None = None prover_link: str | None = None + message: str | None = Field( + default=None, + description="Human-readable explanation of a non-GOOD outcome (e.g. the fuzzer's " + "counterexample / failed-assertion message). Diagnostics only; may be absent.", + ) @property def ref(self) -> RuleRef: @@ -122,6 +130,43 @@ class GaveUpComponent(BaseModel): properties: list[PropertyFormulation] +class DraftedProperty(PropertyFormulation): + """A curtailed component's property the author claims to have encoded, with the units it + named at publish. Unverified: the publish gates were lifted, so the claim was never checked + against a judge or a verification run.""" + units: list[RuleName] = Field( + default_factory=list, + description="The rule/test names the author declared for this property — an unchecked claim.", + ) + + +class CurtailedSkip(PropertyFormulation): + """A curtailed component's property the author explicitly skipped (typically citing the + budget) before publishing what remained.""" + reason: str + + +class CurtailedComponent(BaseModel): + """A component whose formalization the run budget cut short. Whatever it published was + accepted with the validation gates lifted, so neither the encoding nor any verification + result is reliable: the component contributes nothing to ``properties``/``rules``/``groups`` + and is reported in the budget appendix instead. Its inferred properties are partitioned by + disposition: ``drafted`` (claimed encoded, unverified), ``skipped`` (explicitly declined, + with reason), ``unattempted`` (never reached).""" + component: ComponentName + #: Project-relative path of the quarantined partial encoding; None when the run was cut off + #: before anything was published. + artifact: str | None = None + #: Last verification-run link the partial result carried, if any (context only — its + #: outcome predates the final encoding and proves nothing about it). + run_link: str | None = None + #: Optional account of the termination (hard-stop message / the author's own words). + detail: str | None = None + drafted: list[DraftedProperty] = Field(default_factory=list) + skipped: list[CurtailedSkip] = Field(default_factory=list) + unattempted: list[PropertyFormulation] = Field(default_factory=list) + + class PropertyGroup(BaseModel): """An audit-level "P-NN" heading: a synthesized claim over a set of `FormalizedProperty`s (its ``members``, by identity). Members partition — each property belongs to exactly one group — @@ -147,6 +192,7 @@ class CoverageReport(BaseModel): rules_spanning_multiple_groups: list[RuleName] = Field(default_factory=list) skipped_count: int = 0 gave_up_component_count: int = 0 + curtailed_component_count: int = 0 dropped_orphan_rules: int = 0 warnings: list[str] = Field(default_factory=list) @@ -170,10 +216,31 @@ class SourceEditRecord(BaseModel): cumulative_diff: str -type ReportBackend = Literal["prover", "foundry"] +class VerificationArtifactRecord(BaseModel): + """A verification-supporting file a plugin's tool produced during one component's + formalization — a Lean proof discharging instrumented lemmas, an auxiliary + certificate, etc. The content lives on disk at ``path`` (project-relative, written + by the artifact store); the record carries provenance and a description for the + deliverable. Deliberately its own model: report.json is a persisted contract.""" + component: ComponentName + #: The contributing plugin's id (its entry-point name). + plugin: str + #: File basename as registered by the tool. + name: str + #: Open vocabulary, e.g. "lean-proof". + kind: str + description: str + path: str + + +type ReportBackend = Literal["prover", "foundry", "none"] """Which pipeline produced this report. Provenance only — every backend fills the same fields; -this tag just lets the renderer pick the right outcome labels ("Verified" vs "Successful test") -for a report.json it reads cold.""" +this tag just lets the renderer pick the right outcome labels ("Verified" vs "Successful test" +vs "Unverified") for a report.json it reads cold. The producers are the CVL prover (``"prover"``), +Foundry (``"foundry"``), and ``"none"`` — a pipeline that records properties without verifying them +(the analysis-only null backend, ``composer.spec.solana.null_backend``), whose reports are all +UNKNOWN and say so. The set is closed: every backend lives in this repo, so a verification backend +adds its own literal here, plus its wording in ``report/render.py``.""" # --------------------------------------------------------------------------- @@ -251,9 +318,15 @@ class AutoProverReport(BaseModel): #: Formalization gaps — properties that exist but no rule formalizes (see the two gap types). skipped: list[SkippedClaim] = Field(default_factory=list) gave_up_components: list[GaveUpComponent] = Field(default_factory=list) + #: Components the run budget cut short — excluded from every table above, rendered as an + #: appendix. + curtailed_components: list[CurtailedComponent] = Field(default_factory=list) #: Source modifications each component's verification ran against; empty when every #: component was verified against the on-disk source. source_edits: list[SourceEditRecord] = Field(default_factory=list) + #: Verification-supporting artifacts registered by plugin tools during + #: formalization (Lean proofs et al.), written to disk by the artifact store. + verification_artifacts: list[VerificationArtifactRecord] = Field(default_factory=list) coverage: CoverageReport #: Violated rules surfaced as audit issues (one per BAD rule; empty #: when nothing is violated, when synthesis was unavailable, or for a non-prover backend). Prose diff --git a/composer/spec/source/report_prover.py b/composer/spec/source/report_prover.py index e1097464..cabcc4e1 100644 --- a/composer/spec/source/report_prover.py +++ b/composer/spec/source/report_prover.py @@ -12,7 +12,7 @@ from prover_output_utility.models import CheckResult, NodeStatus from composer.spec.cvl_generation import GeneratedCVL -from composer.spec.source.report.collect import ReportComponentInput, Verdict, VerdictFetcher +from composer.spec.source.report.collect import Formalized, Verdict, VerdictFetcher from composer.spec.source.report.schema import Outcome, RuleName _log = logging.getLogger(__name__) @@ -45,18 +45,20 @@ def _fetch(api: ProverOutputAPI, link: str) -> dict[RuleName, Verdict]: c.duration or None, Path(loc.file).name if (loc and loc.file) else None, ) - verdicts[c.rule_name] = cand.merge(verdicts.get(c.rule_name)) + name = RuleName(c.rule_name) + verdicts[name] = cand.merge(verdicts.get(name)) return verdicts def make_prover_fetcher(api: ProverOutputAPI | None = None) -> VerdictFetcher[GeneratedCVL]: """A `VerdictFetcher` that pulls per-rule verdicts from ProverOutputUtility, keyed by each - component's run link. POU calls run off the event loop (one blocking call per run).""" + component's run link. POU calls run off the event loop (one blocking call per run). Only ever + invoked for delivered results (collect skips gave-up / curtailed inputs).""" api = api or ProverOutputAPI() - async def fetch(inp: ReportComponentInput[GeneratedCVL]) -> dict[RuleName, Verdict]: - if inp.formalized is None or inp.formalized.run_link is None: + async def fetch(formalized: Formalized[GeneratedCVL]) -> dict[RuleName, Verdict]: + if formalized.run_link is None: return {} - return await asyncio.to_thread(_fetch, api, inp.formalized.run_link) + return await asyncio.to_thread(_fetch, api, formalized.run_link) return fetch diff --git a/composer/spec/source/source_env.py b/composer/spec/source/source_env.py index a524dfd8..28364331 100644 --- a/composer/spec/source/source_env.py +++ b/composer/spec/source/source_env.py @@ -8,6 +8,7 @@ from graphcore.graph import Builder from graphcore.tools.vfs import GlobalExcludeArg, fs_tools +from composer.pipeline.ecosystem import Ecosystem from composer.spec.tool_env import BaseSourceTools from composer.spec.services import build_rag_tool_env, RAGInputs from composer.spec.service_host import ModelProvider, ServiceHost, Sort @@ -35,6 +36,7 @@ def build_source_tools( store: BaseStore, cache_ns: tuple[str, ...], recursion_limit: int, + ecosystem: Ecosystem, ) -> tuple[BaseTool, ...]: """Wrap the base source tools with the indexed code_explorer sub-agent + the document-ref retrieval tool. Returns the full source tool tuple. @@ -65,6 +67,7 @@ class _ExplorerEnv: llm=models.llm_lite(), ), recursion_limit=recursion_limit, + explorer_prompt=ecosystem.code_explorer_prompt, ) return s.base_source_tools + ( explorer_tool, @@ -81,6 +84,7 @@ class SourceParams(RAGInputs): def build_source_env( *, sort: Sort = "existing", + ecosystem: Ecosystem, **params: Unpack[SourceParams], ) -> ServiceHost: """Build a fully-bound ``ServiceHost`` with both RAG and source tool @@ -97,6 +101,7 @@ def build_source_env( params["store"], params["source_question_ns"], recursion_limit=params["recursion_limit"], + ecosystem=ecosystem, ) return ServiceHost( models=rag_env.models, diff --git a/composer/spec/source/summarizer.py b/composer/spec/source/summarizer.py index c719360c..2cdfd8e2 100644 --- a/composer/spec/source/summarizer.py +++ b/composer/spec/source/summarizer.py @@ -24,7 +24,8 @@ from composer.spec.graph_builder import bind_standard, run_to_completion from composer.cvl.tools import get_cvl, put_cvl, put_cvl_raw, edit_cvl from composer.spec.gen_types import CVLResource, SUMMARIES_DIR, under_project -from composer.spec.context import WorkflowContext, SourceCode, CacheKey +from composer.spec.context import WorkflowContext, SourceCode +from composer.spec.key_family import KeyFamily from composer.spec.util import temp_certora_file, string_hash, ensure_dir from composer.spec.service_host import ServiceHost from composer.spec.source.harness import ContractSetup, ExternalInterface, HarnessDef @@ -287,9 +288,10 @@ class _SummaryCache(BaseModel): content: str -def _summary_key(d: ContractSetup) -> CacheKey[None, _SummaryCache]: - cacher = string_hash(d.model_dump_json())[:16] - return CacheKey("summary-" + cacher) +def _summary_key(d: ContractSetup) -> str: + return "summary-" + string_hash(d.model_dump_json())[:16] + +SUMMARY_KEY = KeyFamily(type(None), _SummaryCache, _summary_key) # --------------------------------------------------------------------------- # Agent @@ -319,7 +321,7 @@ async def setup_summaries( CVLResource pointing to the generated ``custom_summaries.spec`` file. """ - summary_context = ctx.child(_summary_key(config)) + summary_context = ctx.child(SUMMARY_KEY(config)) custom_summaries_path = SUMMARIES_DIR / "custom_summaries.spec" # project-root-relative result_path = under_project(source.project_root, custom_summaries_path) ensure_dir(result_path.parent) diff --git a/composer/spec/system_analysis.py b/composer/spec/system_analysis.py index ad5aa143..54ed4682 100644 --- a/composer/spec/system_analysis.py +++ b/composer/spec/system_analysis.py @@ -13,6 +13,7 @@ from composer.spec.service_host import ServiceHost, Sort from composer.spec.util import slugify_filename from composer.tools.thinking import RoughDraftState, get_rough_draft_tools +from composer.diagnostics.budget import budget_monitor DESCRIPTION = "Component analysis" @@ -160,7 +161,8 @@ def _validation_wrapper( [memory, *get_rough_draft_tools(AnalysisState), *env.analysis_tools] ).inject( lambda g: initial_template.bind(prompt_params).render_to(g.with_initial_prompt_template) - ) + ).with_monitor(budget_monitor()) + graph = b.compile_async() inputs : list[str | dict] = [] diff --git a/composer/spec/system_model.py b/composer/spec/system_model.py index 017be514..ba3164fe 100644 --- a/composer/spec/system_model.py +++ b/composer/spec/system_model.py @@ -17,7 +17,7 @@ class FeatureUnit(Protocol): keys, task ids, labels, and context tags ecosystem-agnostic.""" @property - def display_name(self) -> str: + def display_name(self) -> ComponentName: """Human label for tasks / report rows.""" ... @@ -279,7 +279,7 @@ def slugified_name(self) -> str: # -- FeatureUnit protocol (the ecosystem-agnostic view the driver consumes) --------- @property - def display_name(self) -> str: + def display_name(self) -> ComponentName: return self.component.name @property diff --git a/composer/spec/types.py b/composer/spec/types.py index e7a21d30..49d71f80 100644 --- a/composer/spec/types.py +++ b/composer/spec/types.py @@ -1,3 +1,4 @@ +from dataclasses import dataclass from typing import TYPE_CHECKING, Protocol, Literal # Nominal ``str`` subtypes for the distinct identity fields of an analyzed @@ -15,34 +16,41 @@ # identifier when the design doc names the entity that way, but allowed to be # anything human-readable. # +# ``CheckName``: the backend's name for one check — a CVL rule, a foundry +# test, a fuzz harness function. ``FormalResult.property_checks()`` maps each +# property title onto the checks that verify it. +# ``ComponentName``: human name of an AIComposer component (e.g. "Increment"), +# or "Structural Invariants". +# ``PropertyTitle``: a property's unique snake_case title — the key in a +# component's ``property_rules`` mapping. +# # ``SolidityIdentifier`` and ``RustIdentifier`` are **siblings** under -# ``SourceIdentifier``; the conceptual names are siblings of each other and of -# ``SourceIdentifier``. Passing one where a sibling is expected is a type error, -# even though all are ``str`` at runtime. +# ``SourceIdentifier``; every other name is a sibling of the rest. Passing one +# where a sibling is expected is a type error, even though all are ``str`` at +# runtime. if TYPE_CHECKING: class SourceIdentifier(str): ... class SolidityIdentifier(SourceIdentifier): ... class RustIdentifier(SourceIdentifier): ... class ContractName(str): ... class ProgramName(str): ... + class CheckName(str): ... + class ComponentName(str): ... + class PropertyTitle(str): ... else: SourceIdentifier = str SolidityIdentifier = str RustIdentifier = str ContractName = str ProgramName = str + CheckName = str + ComponentName = str + PropertyTitle = str -type UnitName = str - -type RuleName = UnitName -"""A CVL rule/invariant identifier as it appears in the prover report and in a component's -``property_rules`` mapping.""" - -type ComponentName = str -"""Human name of an AIComposer component (e.g. "Increment"), or "Structural Invariants".""" - -type PropertyTitle = str -"""A property's unique snake_case title — the key in a component's ``property_rules`` mapping.""" +#: A ``CheckName`` as the CVL/prover side speaks it: a rule/invariant identifier as it appears in +#: the prover report and in a component's ``property_rules`` mapping. The same type, not a +#: sibling — a rule name is what the Rust seam calls a check name. +RuleName = CheckName class ArtifactIdentifier(Protocol): @property @@ -52,7 +60,7 @@ def stem(self) -> str: ... def artifact_file(self) -> str: ... class FormalResult(Protocol): - def property_units(self) -> list[tuple[PropertyTitle, list[UnitName]]]: ... + def property_checks(self) -> list[tuple[PropertyTitle, list[CheckName]]]: ... @property def commentary(self) -> str: ... @@ -60,6 +68,20 @@ def commentary(self) -> str: ... @property def artifact_text(self) -> str: ... + +@dataclass(frozen=True) +class Curtailed[T]: + """A formalization the run budget cut short. ``partial`` is whatever the author published + after the budget monitor lifted its validation gates — the raw backend result at the author + boundary, the persisted result inside a pipeline outcome — or ``None`` when the run stopped + (or the author gave up under the wrap-up order) before publishing anything. Either way the + component's encoding and verification state are unreliable: it is not a delivery, is never + cached, and the report keeps it out of the property grouping, surfacing it in the budget + appendix instead. ``detail`` optionally carries context (the hard-stop message, or the + author's own account).""" + partial: T | None + detail: str | None = None + from pydantic import BaseModel, Field type PropertyType = Literal["attack_vector", "safety_property", "invariant"] @@ -85,6 +107,20 @@ class PropertyFormulation(UntitledPropertyFormulation): """ A property or invariant that must hold for the component """ - title: str = Field(description="A short, descriptive snake_case identifier for the property (e.g. 'total_supply_preserved'). Must be unique within the batch of properties.") - + title: PropertyTitle = Field(description="A short, descriptive snake_case identifier for the property (e.g. 'total_supply_preserved'). Must be unique within the batch of properties.") + + +class VerificationArtifact(BaseModel): + """A verification-supporting file produced by a plugin's contributed tool — a + Lean proof discharging instrumented lemmas, an auxiliary certificate, etc. + Registered with its *content*, not a path: the artifact store owns the + deliverable layout and decides where it lands on disk.""" + #: File basename (the store sanitizes to a basename and namespaces by + #: unit and plugin, so collisions across tools are impossible). + name: str + #: Open vocabulary tag, e.g. "lean-proof". + kind: str + #: One-or-two-line blurb for the report deliverable. + description: str + content: str diff --git a/composer/spec/util.py b/composer/spec/util.py index fd1860b0..f3c5d299 100644 --- a/composer/spec/util.py +++ b/composer/spec/util.py @@ -4,7 +4,7 @@ import re import uuid from pathlib import Path, PurePath -from typing import Iterator +from typing import Iterator, Sequence from composer.spec.gen_types import CERTORA_DIR @@ -13,6 +13,15 @@ def string_hash(s: str) -> str: return hashlib.sha256(s.encode()).hexdigest()[:16] +def combine_digests(digests: Sequence[str]) -> str | None: + """Fold digests into one fixed-width value for keying a cache on a *list* of things; + ``None`` for an empty sequence. Order-sensitive: a different order is a different + prompt (or manifest) and must not share a cache entry.""" + if not digests: + return None + return string_hash("|".join(digests)) + + def slugify_filename(name: str) -> str: # Collapse any run of filesystem-unsafe characters into a single underscore so the # result is safe to use as a filename component; falls back to "unnamed" if empty. diff --git a/composer/templates/authoring_protocol.j2 b/composer/templates/authoring_protocol.j2 new file mode 100644 index 00000000..5c286892 --- /dev/null +++ b/composer/templates/authoring_protocol.j2 @@ -0,0 +1,51 @@ +{#- + The host-owned half of an authoring session's system prompt: the tool protocol and what the + publish gate requires. A backend supplies the domain half and this is prepended, so no backend + restates the protocol (and none of them drift from it). + + Context variables: + gate_tool — the tool that builds/checks the spec and stamps the publish gate + has_judge — a reviewer must accept the draft before it can be published + has_checks — the session declares a property→checks mapping and runs it (false for a shared + setup artifact, which formalizes no properties of its own) + check_noun — what one check is called ("check", "rule", "test", "harness function") +-#} +You are authoring a specification. You work on a single spec buffer, iterating on it with tools +until it is ready to publish. + +**The buffer.** `put_spec` replaces the whole buffer; `edit_spec` replaces one exact span within it +and is dramatically cheaper — prefer it once a draft exists. `get_spec` reads the buffer back. Do +not paste the whole spec into your messages; it lives in the buffer. + +**The gate.** `{{ gate_tool }}` is what decides whether your spec is acceptable. Its result is the +ground truth: a failure is something to fix, not something to argue with. Its approval is recorded +against the *exact* buffer it saw, so any later edit invalidates it and you must run it again. +{% if has_checks %} +**What gets checked.** `map_checks` declares which {{ check_noun }}s in your spec verify which +property. `{{ gate_tool }}` runs only the names you list there — a {{ check_noun }} that is in the +spec but not mapped is not run. Declare it before you validate, and re-declare it whenever the set +of {{ check_noun }}s changes. A property may need several {{ check_noun }}s and one {{ check_noun }} +may carry several properties. A {{ check_noun }} you name but did not actually write does not +quietly pass: it comes back as not-run and the gate refuses. + +**Skips.** If a property genuinely cannot be formalized here, `record_skip` it with a justification +rather than writing a {{ check_noun }} that only appears to check it. A skip is reviewed like +anything else. `unskip_property` reverses one. + +**Expected failures.** A {{ check_noun }} whose failure is the *finding* — a real counterexample, a +property that does not in fact hold — is marked with `expect_check_failure` and a reason. Marking it +lets the gate pass with the failure recorded and explained, which is the honest outcome. Do not mark +a {{ check_noun }} whose failure is your own bug. +{% endif %} +{% if has_judge %} +**Review.** `feedback_tool` sends the current draft to a reviewer. Address what it raises, or, when +a prior-round suggestion was tried and provably does not work, file a `rebuttal` with the concrete +evidence — tool output beats argument. Publishing requires an acceptance of the current draft. +{% endif %} +**Publishing.** Call `result` when {% if has_judge %}both the gate and the reviewer have accepted{% +else %}the gate has accepted{% endif %} the buffer as it now stands. `result` is refused if anything +is stale{% if has_checks %}, or if your declared mapping does not account for every property that +was not skipped{% endif %} — that is the gate working, not an error to route around. + +**Giving up.** If the task cannot be completed, `give_up` with the reason. That is a real outcome +and it is reported. It is better than publishing something that only looks checked. diff --git a/composer/templates/autoprove_report.html.j2 b/composer/templates/autoprove_report.html.j2 index f6c87939..0dfcebcb 100644 --- a/composer/templates/autoprove_report.html.j2 +++ b/composer/templates/autoprove_report.html.j2 @@ -229,6 +229,32 @@ section.gaps h4 { } ul.claims { margin: 4px 0 0 18px; padding: 0; font-size: 13px; } ul.claims li { margin-bottom: 4px; } +.finding { + margin-top: 6px; + padding: 4px 8px; + background: var(--bg-pre); + border-radius: 3px; + font-family: ui-monospace, SFMono-Regular, "SF Mono", Menlo, monospace; + font-size: 12px; + white-space: pre-wrap; + word-break: break-word; +} +section.gaps .desc { color: var(--muted-fg); margin: 6px 0 12px; font-size: 13px; } +section.gaps p.note { margin: 4px 0 8px; font-size: 13px; } +section.gaps details { margin: 8px 0 14px; } +section.gaps details > summary { + cursor: pointer; + font-size: 12.5px; + color: var(--muted-fg); + user-select: none; +} +code.unit { + font-family: ui-monospace, SFMono-Regular, "SF Mono", Menlo, monospace; + font-size: 12px; + background: var(--bg-pre); + padding: 1px 5px; + border-radius: 3px; +} .sort-tag { display: inline-block; padding: 1px 7px; @@ -376,6 +402,7 @@ pre.diff .d-file { color: var(--muted-fg); font-weight: 600; }
    {% for d in row.descriptions %}
  • {{ d }}
    • {% endfor %}
{%- elif row.descriptions %}{{ row.descriptions[0] }} {%- else %}(no inferred description){%- endif %} + {%- if row.message %}
{{ row.message }}
{%- endif %} {%- if has_links %} @@ -420,6 +447,44 @@ pre.diff .d-file { color: var(--muted-fg); font-weight: 600; } {%- endif %} +{%- if curtailed %} +
+

Appendix: cut short by the run budget ({{ curtailed | length }})

+

The run's budget ran out while these components were being formalized. Anything + they published was accepted without the usual validation gates, so neither the encodings nor + their verification results are reliable; nothing here contributes to the property groups, + outcome counts, or coverage figures above.

+ {%- for c in curtailed %} +

{{ c.component }} {{ c.status_label }}

+

{{ c.summary }}

+ {%- if c.artifact %} +

Unvalidated draft kept for inspection: {{ c.artifact }} + {%- if c.link.href %} · last {{ c.link.label }}{% endif %}

+ {%- endif %} + {%- if c.detail %}

{{ c.detail }}

{% endif %} +
+ Per-property breakdown ({{ c.rows | length }}) + + + + {%- for row in c.rows %} + + + + + + {%- endfor %} + +
PropertyDispositionNotes
{{ row.description }} {{ row.sort }}{{ row.label }} + {%- if row.units %}{% for u in row.units %}{{ u }}{% if not loop.last %}, {% endif %}{% endfor %} + {%- elif row.note %}{{ row.note }} + {%- else %}—{% endif %} +
+
+ {%- endfor %} +
+{%- endif %} + {%- if source_edits %}

Source modifications

@@ -453,6 +518,9 @@ pre.diff .d-file { color: var(--muted-fg); font-weight: 600; } {%- if coverage.rules_spanning_multiple_groups %} {{ coverage.rules_spanning_multiple_groups | length }} {{ terms.unit_singular }}(s) underwrite multiple high-level properties. {%- endif %} + {%- if coverage.curtailed_component_count %} + {{ coverage.curtailed_component_count }} component(s) were cut short by the run budget (see the appendix). + {%- endif %} diff --git a/composer/templates/cex_analyzer_aggregator_system.j2 b/composer/templates/cex_analyzer_aggregator_system.j2 index e8be92c4..ddaaa3b9 100644 --- a/composer/templates/cex_analyzer_aggregator_system.j2 +++ b/composer/templates/cex_analyzer_aggregator_system.j2 @@ -41,7 +41,7 @@ construct mean, why would this annotation produce a havoc, etc.), and the source-reading tools (`get_file`, `grep_files`, `list_files`) to cross-check a claim from an input cause when you need to confirm same-construct-or-not. -{% include "prover_warning_partial.j2" %} +{% include "prover_warning_fragment.j2" %} **Important**: The input root cause analyses and all of their conclusions are established facts. Do *NOT* second guess them, try to verify these conclusions, diff --git a/composer/templates/cex_analyzer_per_rule_system.j2 b/composer/templates/cex_analyzer_per_rule_system.j2 index 92725b43..9bc75298 100644 --- a/composer/templates/cex_analyzer_per_rule_system.j2 +++ b/composer/templates/cex_analyzer_per_rule_system.j2 @@ -43,7 +43,7 @@ of which control how the Prover reasons about the smart contract's execution, an the counter example it produces. You should use the `cvl_researcher` subagent to help understand the input CVL. -{% include "prover_warning_partial.j2" %} +{% include "prover_warning_fragment.j2" %} Importantly, you are NOT responsible for proposing how to FIX this issue. Your role is simply to explain the counterexample and identify the root cause. diff --git a/composer/templates/code_explorer/common_fragment.j2 b/composer/templates/code_explorer/common_fragment.j2 new file mode 100644 index 00000000..a4422014 --- /dev/null +++ b/composer/templates/code_explorer/common_fragment.j2 @@ -0,0 +1,16 @@ +You have access to file tools (list_files, get_file, grep_files) to explore the project. + +Your job is to answer a specific question about the codebase thoroughly and precisely. + +Guidelines: +- Ground every claim in what you find in the source code. +- If the question asks about behavior, trace through the actual implementation rather than speculating. +- Be concise: the caller needs a dense, actionable answer, not a walkthrough of your exploration process. +- If you discover you do not have enough information to fully answer the question, + (e.g., there is a reference to code not available to you) *DO NOT GUESS*. Indicate in your final answer + that you cannot fully answer the question due to incomplete information. + +If asked a question that cannot be answered by simply looking at the code (e.g., about some completely unrelated +topic) you must decline to answer, indicating it is out of scope for what you're capable of answering. + +When complete, deliver your answer via the `result` tool. diff --git a/composer/templates/code_explorer/index_addendum_fragment.j2 b/composer/templates/code_explorer/index_addendum_fragment.j2 new file mode 100644 index 00000000..8be22105 --- /dev/null +++ b/composer/templates/code_explorer/index_addendum_fragment.j2 @@ -0,0 +1,11 @@ +You have access to findings from prior analyses of this codebase. +These findings were produced by earlier agents investigating the same {{ prior_subject | default("source") }} +and are established facts — do not re-derive or re-verify them. + +When prior findings are provided alongside your task: + +1. If a prior finding directly answers your question, use it as-is. Do not rephrase, re-investigate, or "confirm" what has already been established. +2. If prior findings partially address your question, build on them. Use the established facts as your starting point and only investigate what remains unanswered. +3. If no prior findings are relevant, proceed with fresh analysis. + +Prior findings are prefixed with the original question that prompted them so you can judge their relevance to your current task. diff --git a/composer/templates/code_explorer/prior_findings_fragment.j2 b/composer/templates/code_explorer/prior_findings_fragment.j2 new file mode 100644 index 00000000..842af27a --- /dev/null +++ b/composer/templates/code_explorer/prior_findings_fragment.j2 @@ -0,0 +1,5 @@ +{% if prior_findings == "established" %} +{% include "code_explorer/index_addendum_fragment.j2" %} +{% elif prior_findings == "versioned" %} +{% include "code_explorer/versioned_index_addendum_fragment.j2" %} +{% endif %} diff --git a/composer/templates/code_explorer/rust/common_fragment.j2 b/composer/templates/code_explorer/rust/common_fragment.j2 new file mode 100644 index 00000000..9153eae0 --- /dev/null +++ b/composer/templates/code_explorer/rust/common_fragment.j2 @@ -0,0 +1,3 @@ +{% include "code_explorer/common_fragment.j2" %} + +The project is a Rust crate or workspace. Navigate from `Cargo.toml` and the crate modules; quote exact Rust snippets rather than paraphrasing them. diff --git a/composer/templates/code_explorer/solana.j2 b/composer/templates/code_explorer/solana.j2 new file mode 100644 index 00000000..5d8a2c97 --- /dev/null +++ b/composer/templates/code_explorer/solana.j2 @@ -0,0 +1,13 @@ +You are a code-exploration assistant analyzing the Rust source of an on-chain program. + +{% include "code_explorer/rust/common_fragment.j2" %} + +When the question touches program behavior, cite: +- the instruction handlers / entry points and the authorization each one performs (and which perform none) — signer checks, `Signer<'info>`, `has_one`, `#[account(signer)]` +- how state is addressed and stored: account types, PDAs, seeds/bumps, owner checks +- CPIs: which program is invoked and where the callee's identity comes from (a hardcoded program id vs. an unchecked account) + +{% with prior_subject = "source" %} +{% include "code_explorer/prior_findings_fragment.j2" %} +{% endwith %} + diff --git a/composer/templates/code_explorer/solidity.j2 b/composer/templates/code_explorer/solidity.j2 new file mode 100644 index 00000000..71b2fb2e --- /dev/null +++ b/composer/templates/code_explorer/solidity.j2 @@ -0,0 +1,9 @@ +You are a code exploration assistant analyzing smart contract source code. + +{% include "code_explorer/common_fragment.j2" %} + +Quote the relevant function signatures, state variable declarations, or code snippets rather than paraphrasing them. + +{% with prior_subject = "contracts" %} +{% include "code_explorer/prior_findings_fragment.j2" %} +{% endwith %} diff --git a/composer/templates/code_explorer/soroban.j2 b/composer/templates/code_explorer/soroban.j2 new file mode 100644 index 00000000..041b5e52 --- /dev/null +++ b/composer/templates/code_explorer/soroban.j2 @@ -0,0 +1,13 @@ +You are a code-exploration assistant analyzing the Rust source of a Soroban contract. + +{% include "code_explorer/rust/common_fragment.j2" %} + +When the question touches contract behavior, cite: +- entry-point functions and the `require_auth` / `require_auth_for_args` each one performs (and which perform none) +- how state is stored: `instance` / `persistent` / `temporary`, and the `DataKey`s that address it +- cross-contract calls: the callee's address and where it comes from + +{% with prior_subject = "source" %} +{% include "code_explorer/prior_findings_fragment.j2" %} +{% endwith %} + diff --git a/composer/templates/code_explorer/versioned_index_addendum_fragment.j2 b/composer/templates/code_explorer/versioned_index_addendum_fragment.j2 new file mode 100644 index 00000000..8cd726dc --- /dev/null +++ b/composer/templates/code_explorer/versioned_index_addendum_fragment.j2 @@ -0,0 +1,17 @@ +You may be provided with other question/answer pairs that were found to be similar +to the question you are asked. These question/answer pairs *may* have been derived +on a prior version of the codebase that you are exploring now; such pairs will be clearly +marked as being (potentially) out of date. Use the following protocol to use these +prior results effectively: + +1. If a prior finding is *not* marked as out of date, and directly answers the question you are asked, + use that answer as is; do not rephrase, re-investigate, or "verify" the answer +2. If a prior finding is *not* marked as out of date, and *partially* answers the question you are asked, + use that answer as a verified starting point and fill in any missing details. + +If a prior question/answer pair that is marked as (potentially stale) +either completely or partially answers the question posed to you, you *should* +use your source tools to determine if the substantive and relevant details of the answer +are still true on this version of the code. If you verify that these details +remain true, you may reuse (in part or in whole) the existing answer as you would +an up-to-date answer. diff --git a/composer/templates/design_doc_finder_system_prompt.j2 b/composer/templates/design_doc_finder_system_prompt.j2 index f6319d2b..a227b4b2 100644 --- a/composer/templates/design_doc_finder_system_prompt.j2 +++ b/composer/templates/design_doc_finder_system_prompt.j2 @@ -1,7 +1,7 @@ {#- System prompt for the design-document finder sub-agent (see composer/spec/source/design_doc_finder.py). Composed from the shared role + - source-tool partials so the tool description stays in lockstep with every other + source-tool fragments so the tool description stays in lockstep with every other source agent. `source_tools_has_explorer` is set false because the finder binds only the bare fs tools (no code_explorer sub-tool). -#} diff --git a/composer/templates/harness_generation_prompt.j2 b/composer/templates/harness_generation_prompt.j2 index f788bce0..e2e60aeb 100644 --- a/composer/templates/harness_generation_prompt.j2 +++ b/composer/templates/harness_generation_prompt.j2 @@ -26,6 +26,14 @@ in `certora/harnesses/TokenInstance1.sol`. The actual harness implementation should contain a minimal contract which simply extends the target contract. You should only add code necessary to satisfy the Solidity compiler/typechecker; e.g., code to invoke the parent contract's constructor. +**Important**: a harness must be a *deployable* contract; the Prover has no instance for a contract the compiler refuses to instantiate. A target declared `abstract` +(or one that leaves any function of an interface it inherits unimplemented) cannot be extended directly for this reason. When the target is such a contract, locate the +concrete contract in the project that inherits from it — the one the protocol actually deploys — and extend *that* instead, keeping the harness itself just as minimal. +Where several concrete contracts inherit from the target, pick the one the protocol's deployment scripts/configuration use. Determining whether a target is deployable +requires looking at its whole inherited interface rather than the target's own file alone: a function may be declared in an interface the target inherits and implemented +by a sibling contract that is only mixed in further down the hierarchy. Only if the project has no concrete contract inheriting from the target should the harness itself +implement the functions that are missing. + The harness contract will need to import the target contract; you should use the relative path of the target file. For example, if `TokenInstance1` is inheriting from `MyToken` declared at the path `src/contracts/token/MyToken.sol`, then `TokenInstance1` should use `import "src/contracts/token/MyToken.sol";`. Do NOT try to use imports with `..`. @@ -36,9 +44,7 @@ Once all harness contracts have been generated, call the `result` tool to delive mapping keyed by each target contract's `solidity_identifier` (from the input list above) to the list of harness descriptions for that target. Each harness description contains the relative path to the generated file and the Solidity identifier you chose for the generated harness. -In addition, specify the Solidity compiler to use when compiling these harnesses. You should specify this as `solcX.Y`, where `X` and `Y` are the major/minor version numbers. -For example, to specify Solidity version 0.8.31, use `solc8.31` as the solidity compiler. -As part of delivering your result your generated harnesses will be checked for syntactic/type validity. Upon rejection, the Solidity compiler error messages will be returned to you; -repair any issues and re-invoke the result tool. +As part of delivering your result your generated harnesses are compiled with the project's own build settings and checked for syntactic/type validity. +Upon rejection, the Solidity compiler error messages will be returned to you; repair any issues and re-invoke the result tool. diff --git a/composer/templates/property_generation_prompt.j2 b/composer/templates/property_generation_prompt.j2 index 03c43137..6adfa134 100644 --- a/composer/templates/property_generation_prompt.j2 +++ b/composer/templates/property_generation_prompt.j2 @@ -76,25 +76,16 @@ feedback as soon as possible. or invariant, but reference your written justification rule + If the above two approaches fail, you may still consider adding `require` statements, but add comments that either document the trust assumption you are making and/or arguing for its Validity -* If the failure is due to a HAVOC from an unresolved call: - + Was the call to a function whose implementation exists in the prover inputs? If so, it is likely the prover - could not resolve the callee contract and link to the extant implementation. In this case, you should consider - using a `DISPATCHER` summary. (with caveats, see below) - + If the call was to an function whose implementation doesn't exist in the scene, add a summary which - (soundly) models the behavior of the called function. In many cases, these summaries will have to model the - (unknown) callee contract's state, you should use "ghost variables" to model this. - + If the call was completely unresolved (no resolved selector/signature), consider using a "dispatch list" - summary. Analyze the location of the unresolved call, and consider what the callee functions might be. - + Do *NOT* try to mark ghost state as `persistent` to work around the HAVOC issue. - + Do *NOT* apply a `NONDET` summary to a side-effecting function to make the HAVOC "go away", this makes - the verification results useless. +* If the failure is due to a HAVOC from an unresolved call: follow the triage ladder described in section B3 of the + "CVL Summarization — Knowledge Base" context document * If the counterexample appears to point to a potential issue in the code that should be reviewed by a security expert, use the "expect fail" tool to mark that the rule failure may be legitimate. * If you have a parametric rule or invariant that is persistently failing on a method, and that method should genuinely be excluded from the property/invariant being proven, use a `filtered` block to exclude it. + You should not use this strategy lightly; only do so when you are 95% certain that the method in question should be excluded from the property. -* If the property or invariant is genuinely incorrect or actually invalid you may mark the property as "skipped" and + + Consult section 4 of the "CVL Invariants and Quantifiers" context document for the policy you should apply when skipping +* If the property or invariant is genuinely incorrect or actually invalid, you may mark the property as "skipped" and remove your attempts to formalize it from the spec * If the rule is failing due to a `SANITY_FAILURE`, this is due to the assertion in the rule/invariant being unreachable. In other words, the preconditions in the rule and the logical encoding of the implementation's code are themselves @@ -104,26 +95,29 @@ feedback as soon as possible. adaptive thinking and the cvl researcher to come to *principled* solutions. Do *NOT* try to brute force the solution by trying random changes. -#### `DISPATCHER` Caveats - -The special `DISPATCHER` summary *unsoundly* assumes that the callee contract for an unresolved call exists in the prover -input. This is acceptable if the callee contract is *definitely* one of the contracts in prover inputs. However, -for something like an ERC20 token; the prover input may itself have some ERC20 token implementations in its inputs -(e.g., a special token representing a voting share). If you use a `DISPATCHER` in this case, you will unsoundly -assume *EVERY* token interacted with by *EVERY* contract must be one of the contracts in the prover inputs. - -Unless you can conclude that the DISPATCHER assumption is justified, use a hybrid summarization approach: -within the generated summary, match the callee contract against known implementers in the prover input. -For those callees, simply dispatch the call to the relevant implementation. For calls "outside" of the prover -inputs, fall back on a sound model of the external behavior, using e.g., ghosts. - ## Step 4 Once all non-skipped properties have been approved by the feedback judge, AND all rules that are not expected as failing have been verified with the prover, deliver your result with the `result` tool. In order -for your result to be accepted, you *must* have run the `verify_spec` tool *without* any `rule` arguments on -the most recent version of the spec. Similarly, you *must* have received positive feedback from the feedback judge -on the most recent change to your spec. +for your result to be accepted, you *must* have run the `verify_spec` tool one or more times +so that *all* rules/invariants in the specification file have been "covered" by at least one run +of the prover. A rule/invariant is "covered" by a `verify_spec` +invocation if the `exclude_rules`/`rule` flags you provide cause the rule/invariant to be checked +by the prover. For example, `verify_spec(rule=["some_rule"])` followed by `verify_spec(exclude_rules=["some_rule"])` +would cover all rules. Further, any rule/invariant not marked as "expected to fail" must +be reported as `VERIFIED` in these most recent runs. NB That expected to fail rules must +are not exempt from this coverage requirement. + +A single, final prover run that covers all rules should be your default mode. +Splitting rules across multiple runs should only be used if one or more rule is particularly expensive in +terms of solving time making it infeasible to verify alongside others in the spec file within the time limit of the prover. + +*Important*: If you change your state (via the skip declarations, spec edits, etc.) any validation results from your +prover runs are *invalidated*. + +Similarly, you *must* have received positive feedback from the feedback judge on the most recent change to your spec. +Like the prover tool, any changes to your state (include the spec, skip declarations, etc.) will invalidate +the positive feedback result, and will necessitate "restamping". The `result` tool *also* requires a `property_rules` mapping: for every property you did NOT skip (referenced by its unique snake_case title from the batch listing above), list the name(s) of the rule(s)/invariant(s) in your @@ -171,6 +165,25 @@ You may also make use of the following CVL resources, available as CVL files. {% endfor %} {% endif %} +{% if hostile_candidates %} +## Prover-hostile functions in this scene (summarization candidates) + +A static analysis of the sanity run flagged the functions below as prover-hostile — they survive +optimization and/or dominate the difficulty, so left inlined they are a likely timeout source. Where a +property does NOT depend on a function's exact result, summarizing it is often the difference between a +proof and a timeout. Only a `curated` entry carries a concrete, vetted summary; for the rest you must +choose the summary that is sound for the property at hand (and justify it). This is a hint, not a +directive — a function you genuinely need exact should stay unsummarized. + +{% for c in hostile_candidates %} +- `{{ c.function }}`{% if c.get('signature') %} `{{ c.signature }}`{% endif %} [{{ c.signals | join(", ") }}]{% if c.get('mutating') %} (state-changing){% endif %}{% if c.get('file') %} — {{ c.file }}{% if c.get('line') %}:{{ c.line }}{% endif %}{% endif %} + {{ c.evidence }} + {% if c.get('reaching_count') %}reaches {{ c.reaching_count }} entry method(s){% endif %} + {% if c.get('candidate_summary') %}suggested summary (curated): {{ c.candidate_summary }}{% endif %} + {% if c.get('boundaries') %}can also be summarized at a caller: {% for b in c.boundaries %}`{{ b.signature }}`{% if not loop.last %}, {% endif %}{% endfor %}{% endif %} +{% endfor %} +{% endif %} + Use the available memory tools to track your progress through this algorithm, any important lessons about CVL you may have learned, or any other significant, relevant information to this task. In particular, be sure to update your memory before calling the prover to summarize diff --git a/composer/templates/property_generation_system_prompt.j2 b/composer/templates/property_generation_system_prompt.j2 index ee4a4b51..7c85f0df 100644 --- a/composer/templates/property_generation_system_prompt.j2 +++ b/composer/templates/property_generation_system_prompt.j2 @@ -7,6 +7,22 @@ {% include "cvl_edit_guidance.j2" %} +## Running the Prover + +The `verify_spec` tool runs the Certora prover on the current version of your spec against the +source under verification, and reports a verdict (`VERIFIED`, `VIOLATED` with counterexample +analysis, `TIMEOUT`, etc.) for each rule and invariant it ran. + +Its two optional arguments select which of the spec's declared rules/invariants the run executes; +they are mutually exclusive: + +- **`rules`** — run only the named rules/invariants. +- **`exclude_rules`** — run every declared rule/invariant *except* the named ones. + +With neither argument, the run executes every rule and invariant declared in the spec. Names in +either list refer to the declarations in the current spec. `verify_spec` must be the only tool +call in its turn. + {% if source_editing %} ## Editing the Source Under Verification diff --git a/composer/templates/prover_warning_partial.j2 b/composer/templates/prover_warning_fragment.j2 similarity index 100% rename from composer/templates/prover_warning_partial.j2 rename to composer/templates/prover_warning_fragment.j2 diff --git a/composer/templates/resume_prompt.j2 b/composer/templates/resume_prompt.j2 index 66e9b3bf..c760ad80 100644 --- a/composer/templates/resume_prompt.j2 +++ b/composer/templates/resume_prompt.j2 @@ -1,8 +1,4 @@ -The workflow you are resuming was started using the following spec file: - -``` -{{ orig_spec }} -``` +The workflow you are resuming was started with the specification(s) described below. When the initial workflow was completed, you summarized your progress as follows: @@ -10,18 +6,22 @@ When the initial workflow was completed, you summarized your progress as follows {{ commentary }} ``` -However, the specification file has since been updated to the following: +{% for d in spec_deltas %} +The spec at `{{ d.vfs_path }}` was originally: ``` -{{ new_spec }} +{{ d.orig }} ``` -This NEW version of the specification has already been changed in the VFS at the path `rules.spec`. +It has since been updated to the following (already applied to the VFS at `{{ d.vfs_path }}`): -You must carefully analyze the differences between the specifications. Identify what -has been added, removed or changed, and consider what, if anything, in the current version -of the implementation has been changed. +``` +{{ d.new }} +``` +Carefully analyze the differences between these specifications. Identify what has been added, +removed, or changed, and consider what, if anything, in the current implementation must change. +{% endfor %} {% for change in other_changes %} In addition, the {{ change.single_form }} used in the original workflow was the following: @@ -43,10 +43,9 @@ In addition to the above changes, you must also carefully analyze the difference Identify what has been added, removed or changed, and consider what, if anything, in the current version of the implementation has been changed. {% endfor %} - {% if spec_change_commentary %} The following is provided as commentary on the input changes, which can include the reasons for the changes or a summary thereof: {{ spec_change_commentary }} -{% endif %} \ No newline at end of file +{% endif %} diff --git a/composer/templates/rust/_vulnerability_patterns.j2 b/composer/templates/rust/vulnerability_patterns_fragment.j2 similarity index 100% rename from composer/templates/rust/_vulnerability_patterns.j2 rename to composer/templates/rust/vulnerability_patterns_fragment.j2 diff --git a/composer/templates/solana/analysis_system.j2 b/composer/templates/solana/analysis_system.j2 index 4b202349..8e611d8c 100644 --- a/composer/templates/solana/analysis_system.j2 +++ b/composer/templates/solana/analysis_system.j2 @@ -8,7 +8,7 @@ the application. {% with draft_subject = "your response" %} {% include "rough_draft_protocol.j2" %} {% endwith %} -{% with source_tools_has_explorer = sort == "existing" %} +{% with source_tools_has_explorer = sort == "existing", source_language = "Rust" %} {% include "source_tools_system_prompt.j2" %} {% endwith %} When reading Rust/Anchor source, pay attention to the instruction handlers, their diff --git a/composer/templates/solana/property_prompt.j2 b/composer/templates/solana/property_prompt.j2 index b2255daa..83112e7f 100644 --- a/composer/templates/solana/property_prompt.j2 +++ b/composer/templates/solana/property_prompt.j2 @@ -47,9 +47,9 @@ Security properties fall into three categories: When reasoning about attack vectors and the checks each account requires, consider these vulnerability patterns: -{% include "rust/_vulnerability_patterns.j2" %} +{% include "rust/vulnerability_patterns_fragment.j2" %} -{% include "solana/_vulnerability_patterns.j2" %} +{% include "solana/vulnerability_patterns_fragment.j2" %} NB: you do **NOT** need to formalize the properties yourself; limit your analysis to properties that can reasonably be formalized by the downstream verification tool, as described in your system prompt. diff --git a/composer/templates/solana/property_system.j2 b/composer/templates/solana/property_system.j2 index 877eeb86..656e9353 100644 --- a/composer/templates/solana/property_system.j2 +++ b/composer/templates/solana/property_system.j2 @@ -37,7 +37,7 @@ non-triviality, a clear violation consequence, and defensibility in your `reason ## Source exploration -{% with source_tools_has_explorer = sort == "existing" %} +{% with source_tools_has_explorer = sort == "existing", source_language = "Rust" %} {% include "source_tools_system_prompt.j2" %} {% endwith %} diff --git a/composer/templates/solana/_vulnerability_patterns.j2 b/composer/templates/solana/vulnerability_patterns_fragment.j2 similarity index 100% rename from composer/templates/solana/_vulnerability_patterns.j2 rename to composer/templates/solana/vulnerability_patterns_fragment.j2 diff --git a/composer/templates/soroban/README.md b/composer/templates/soroban/README.md new file mode 100644 index 00000000..30b18757 --- /dev/null +++ b/composer/templates/soroban/README.md @@ -0,0 +1,99 @@ +# Soroban Prompt Templates + +Templates for Soroban/Stellar smart contracts. The structure is similar to +`composer/templates/solana/`, but we use Soroban terminology: contracts, entry points, +`Address` auth, storage kind, and cross-contract calls. + +| File | Role | +|---|---| +| `analysis_system.j2` | System prompt for model extraction | +| `analysis_prompt.j2` | User prompt for extracting the application model | +| `component_context.j2` | Component context rendered into property prompts | +| `property_system.j2` | System prompt for property inference | +| `property_prompt.j2` | User prompt for property inference | +| `platform_model_fragment.j2` | Soroban execution model facts shared by prompts | +| `vulnerability_patterns_fragment.j2` | Soroban-specific bug patterns | + +Rust-level issues are in `rust/vulnerability_patterns_fragment.j2`; Soroban-specific +issues are here. + +## Model + +These templates render against `SorobanApplication`, defined in +[composer/spec/soroban/model.py](../../spec/soroban/model.py) and bound into the seam as `SOROBAN` +in [composer/pipeline/ecosystem.py](../../pipeline/ecosystem.py). That module is authoritative; the +sketch below is a reading aid. Because the four templates are registered in +`template_manifest.json`, [tests/test_fuzzed_templates.py](../../../tests/test_fuzzed_templates.py) +renders each of them against hundreds of generated models — an undefined variable or a field rename +that breaks a template fails CI. + +Fields: + +```text +SorobanApplication: + components: list[SorobanContract | SorobanAuthority] + contracts, authorities + +SorobanContract: + name, contract_identifier, contract_id, description + storage_entries: list[StorageEntry] + functions: list[SorobanFunction] + components: list[ContractComponent] + +StorageEntry: + key + durability: "instance" | "persistent" | "temporary" + value_type + description + +SorobanFunction: + name, description, args, returns + auth: list[AuthRequirement] + storage: list[StorageAccess] + calls: list[ContractCall] + events, errors, requirements + +AuthRequirement: + address + kind: "require_auth" | "require_auth_for_args" + description + +StorageAccessSite: + key + durability + access: "read" | "write" | "remove" | "extend_ttl" + +ContractCall: + target_contract + description + +ContractComponent: + name, description, functions, storage_keys, interactions, requirements + +SorobanAuthority: + name, description, assumptions +``` + +`component_context.j2` expects a resolved context with `app`, `contract`, +`component`, `functions`, `storage_entries`, `sibling_components`, and +`sibling_contracts` — that is `SorobanComponentInstance`, the ecosystem's `Unit`. It resolves the +component's `functions` and `storage_keys` name lists into objects, because a bare storage key +without its durability is not interpretable. `_soroban_validate` rejects a name that would not +resolve, so the templates never have to render a hole. + +## Backend Boundary + +`property_system.j2` includes `{{ backend_guidance }}` — the system prompt, not the initial one, +because the guidance is fixed for a whole run and so belongs inside the cached prefix. For Certora Sunbeam, +that guidance should cover Rust `#[rule]` specs using Cavalier/CVLR macros +(`cvlr_assert!`, `cvlr_assume!`, `cvlr_satisfy!`), nondeterministic inputs, and +`certoraSorobanProver` config. The Soroban templates should identify +security properties; backend guidance decides how to express them. + +## Sources + +- Soroban auth and `Address::require_auth` +- Soroban storage kind, TTL, archive, and restore rules +- SEP-41 token interface and Stellar Asset Contract behavior +- Certora Sunbeam documentation +- Public Soroban audit checklists and detector catalogs diff --git a/composer/templates/soroban/analysis_prompt.j2 b/composer/templates/soroban/analysis_prompt.j2 new file mode 100644 index 00000000..75cab1d0 --- /dev/null +++ b/composer/templates/soroban/analysis_prompt.j2 @@ -0,0 +1,130 @@ +{% from "shared/analysis_macros.j2" import input_phrase with context %} +{%- set impl = "Rust implementation" %} +# Background + +You have been given the {{ input_phrase(impl_noun=impl) }} of a Soroban/Stellar application. +Ground every conclusion only in what you were given. + +{% include "soroban/platform_model_fragment.j2" %} + +# Task + +Analyze the {{ input_phrase(impl_noun=impl) }} and extract a structured model: +implemented **contracts**, exposed **functions**, owned **storage entries** with +storage kind, performed **auth checks**, calls, and external actors. Use +natural language; do not write Rust or pseudo-code. + +## Application Description + +Provide an "Application Type" — a broad category (e.g. "AMM", "Lending market", "Vault", +"Token", "Escrow") — and a short (2–3 sentence) description that refines it. Then a list of +"System Components": each is either a **Contract** or an **External Authority**. + +### External Authorities + +Any actor the application relies on but does not itself implement. For Soroban that +typically means: + +- **An admin or governance address** — whatever `DataKey::Admin` (or its equivalent) holds, and + any address the contract treats as privileged. +- **An off-chain signer** — a price submitter, a relayer, a keeper. +- **A token contract.** Any token the application does not implement: a SEP-41 token, or a + Stellar Asset Contract (SAC) which is the built-in contract that wraps a classic Stellar asset so + it can be used from Soroban. +- **Any other third-party contract** it calls, for example an oracle, a DEX, a router. + +For each authority provide: + +- `name`: a short unique label, as other components will refer to it by this name. +- `description`: what the actor is and what role it plays for this application. +- `assumptions`: what the application is *trusting* it to do or not do — the beliefs that, if + false, break the application. Be concrete. For a SAC, the issuer retains + `mint`, `clawback`, and `set_authorized`, so a balance held in it can change or be frozen + without this application acting; record that here. For an oracle: how fresh, how honest, and + what happens if it stops answering. For an admin: which powers it holds and whether it is + assumed honest, or merely assumed not to be compromised. + +Do not speculate about how an external authority works internally. + +### Contracts + +For each contract this app implements, provide: + +- `name`: a short label (e.g. "Vault", "Pool", "Governor"). +- `contract_identifier`: the `#[contract]` type or crate/module identifier, matching + `[a-zA-Z_][a-zA-Z0-9_]*`. +- `contract_id`: the deployed contract address (StrKey, `C…`) if the implementation pins one, else + null. +- `description`: the contract's role in the application. +- `storage_entries`: the storage this contract owns, described below. +- `functions`: the contract's entry points, described below. +- `components`: the contract's feature groups, described below. + +#### Storage entries + +One entry per logical value read or written: + +- `key`: source spelling, e.g. a `DataKey` variant such as `Balance(Address)` or + a `Symbol`. +- `durability`: storage kind, either `instance`, `persistent`, or `temporary`, + as actually used at the call site. +- `value_type`: the stored type. +- `description`: what the entry means and who may change it. + +Record the durability the code actually uses. Do not "correct" it to one the value seems to deserve. +A balance kept in `temporary` storage (destroyed, unrecoverably, when its TTL lapses) or per-user data kept in +`instance` storage (reloaded in full on every invocation, against a shared size cap) is a finding +for the analysis that follows, and it only survives to that stage if you write down what is +there rather than what should be. + +#### Functions + +For each `#[contractimpl]` entry point: + +- `name`: the function's snake_case name. +- `description`: what it does behaviorally (not how). +- `args`: its arguments (name & type), excluding `env`. +- `returns`: its return type, or null. +- `auth`: each `require_auth` / `require_auth_for_args`, naming the authorized + address argument or stored address (for example, "admin from `DataKey::Admin`"). + If there is no auth, record an empty list. +- `storage`: the storage this function touches. Each entry gives the `key`, + `durability` storage kind, and `access` (`read`, `write`, `remove`, or + `extend_ttl`). +- `calls`: cross-contract calls. Name the target or how its address is + obtained, what is called, and whether failures are handled (`try_*`) or abort. +- `events`: the events it publishes, if any. +- `errors`: `#[contracterror]` variants, `panic_with_error!`, and unhandled aborts + such as `unwrap`, overflow, or failed conversions. +- `requirements`: the function's behavioral specification, stated as requirements ("The function + must ..."). + +#### Components + +Components group contract features. A component is broader than one function +and narrower than the whole contract. Describe what it does. + +Group functions by feature and shared storage, not file layout or call order. +Admin and settings functions usually form their own component. + +For each component provide: + +- `functions`: the names of the functions that make up this component. Every function you declared + for the contract **must appear in at least one component**. A function may appear in more than + one component when it genuinely serves two features. +- `storage_keys`: which of the contract's declared storage entries this component maintains. As + with functions, an entry may be listed by more than one component when they share it. + For example, a balance map read by a "Deposits" component and written by a "Withdrawals" one belongs to + both. +- `requirements`: any requirements or assumptions on the component's behavior, stated as + implementation requirements ("The implementation must ..."). +- `interactions`: how this component uses other components, contracts, or + external actors. Name the other contract/component or external actor. + +## On Interactions + +List contracts and external actors as fully as the inputs allow. Do not guess how +external actors work internally. A call into another contract from this app is an +internal contract/component interaction, not an external actor. + +{% include "shared/analysis_memory.j2" %} diff --git a/composer/templates/soroban/analysis_system.j2 b/composer/templates/soroban/analysis_system.j2 new file mode 100644 index 00000000..63a2ef36 --- /dev/null +++ b/composer/templates/soroban/analysis_system.j2 @@ -0,0 +1,14 @@ +{% from "shared/analysis_macros.j2" import input_phrase with context %} +You are an experienced Soroban/Stellar smart-contract architect. Extract a +structured behavioral model from the {{ input_phrase(impl_noun="Rust implementation") }}. The +model will drive security-property generation. + +{% include "shared/architect_behavior_tools.j2" %} +{% with draft_subject = "your response" %} +{% include "rough_draft_protocol.j2" %} +{% endwith %} +{% with source_tools_has_explorer = sort == "existing", source_language = "Rust" %} +{% include "source_tools_system_prompt.j2" %} +{% endwith %} +Focus on entry points, `Address` auth, `DataKey`/storage kind, TTL calls, +cross-contract call targets and error handling, abort paths, and upgrade paths. diff --git a/composer/templates/soroban/component_context.j2 b/composer/templates/soroban/component_context.j2 new file mode 100644 index 00000000..00a84fa1 --- /dev/null +++ b/composer/templates/soroban/component_context.j2 @@ -0,0 +1,106 @@ +Target component: {{ context.component.name }} in {{ context.contract.name }} +(`{{ context.contract.contract_identifier }}`). + +Component description: + +{{ context.component.description }} + +{% if context.component.requirements %} +Component requirements: +{% for req in context.component.requirements %} +* {{ req }} +{% endfor %} +{% endif %} +{% if context.storage_entries %} + +## The storage this component maintains +{% for entry in context.storage_entries %} +* `{{ entry.key }}` — {{ entry.durability }} storage, `{{ entry.value_type }}`. {{ entry.description }} +{% endfor %} +{% elif context.component.storage_keys %} + +The storage keys this component maintains: {{ context.component.storage_keys | join(", ") }}. +{% endif %} + +Application type: {{ context.app.application_type }}. + +## This component's entry points + +Functions in scope: +{% for fn in context.functions %} + +### `{{ fn.name }}` + +{{ fn.description }} +{% if fn.requirements %} +Requirements: +{% for req in fn.requirements %} +* {{ req }} +{% endfor %} +{% endif %} +Signature: `{{ fn.to_signature() }}` +Authorization: +{% if fn.auth %} +{% for au in fn.auth %} +* `{{ au.address }}` via `{{ au.kind }}` — {{ au.description }} +{% endfor %} +{% else %} +* **No auth.** +{% endif %} +{% if fn.storage %} +Storage accessed: +{% for st in fn.storage %} +* `{{ st.key }}` ({{ st.durability }}) — {{ st.access }} +{% endfor %} +{% endif %} +{% if fn.calls %} +Cross-contract calls: +{% for call in fn.calls %} +* → `{{ call.target_contract }}`: {{ call.description }} +{% endfor %} +{% endif %} +{% if fn.events %} +Events published: {{ fn.events | join(", ") }}. +{% endif %} +{% if fn.errors %} +Failure conditions: +{% for err in fn.errors %} +* {{ err }} +{% endfor %} +{% endif %} +{% endfor %} + +{% if context.sibling_components %} +## The rest of the contract + +Other components of {{ context.contract.name }}. These are provided for context only. Do not +write properties about them. They matter because they may share storage with this component, or +execute before, after, or between its calls: +{% for other in context.sibling_components %} +* **{{ other.name }}**: {{ other.description }} +{% endfor %} +{% endif %} + +{% if context.sibling_contracts %} +Other contracts in the application: +{% for c in context.sibling_contracts %} +* `{{ c.name }}` (`{{ c.contract_identifier }}`): {{ c.description }} +{% endfor %} +{% endif %} +{% if context.app.authorities %} +External authorities / actors the system interacts with: +{% for a in context.app.authorities %} +* {{ a.name }}: {{ a.description }}{% if a.assumptions %} (assumed: {{ a.assumptions | join("; ") }}){% endif %} +{% endfor %} +{% endif %} + +{% if context.component.interactions %} +Component links: +{% for other in context.component.interactions %} +{% if other.authority is defined %} +- External actor "{{ other.authority }}": {{ other.description }} +{% else %} +- The `{{ other.component }}` component of {% if other.contract == context.contract.name %}this contract{% else %}the {{ other.contract }} contract{% endif %}: {{ other.description }} +{% endif %} +{% endfor %} +{% endif %} diff --git a/composer/templates/soroban/platform_model_fragment.j2 b/composer/templates/soroban/platform_model_fragment.j2 new file mode 100644 index 00000000..59662e06 --- /dev/null +++ b/composer/templates/soroban/platform_model_fragment.j2 @@ -0,0 +1,55 @@ +Soroban is Stellar's Rust-to-Wasm smart-contract platform. The host manages +storage, auth, ledger data, and contract calls. + +**Contracts.** A `#[contract]` type exposes `#[contractimpl] pub fn` entry +points, usually with `env: Env` first. Values passed in or out use `#[contracttype]`; +errors use `#[contracterror]`. Initialization is normally an explicit +`initialize`/`init` entry point and must be guarded. + +**Auth.** There is no built-in caller like `msg.sender`. A function acts +for an `Address` only if it calls `addr.require_auth()` or +`addr.require_auth_for_args(...)`. Require auth from the address whose funds, +rights, or settings are affected. The host checks the signed call tree, +including arguments, and handles nonces and replay protection. + +**Address values.** An `Address` may be a Stellar account or a contract. Contract +addresses approve calls through `__check_auth`. When a contract calls another +contract, the caller contract is approved only for that direct call. + +**Storage.** `env.storage().instance()`, `.persistent()`, and `.temporary()` are +separate key spaces. + +- `instance`: contract-instance state, loaded with the instance, suitable only + for small global config. +- `persistent`: independent entries for durable per-user or protocol state. +- `temporary`: cheapest, but expiry deletes the entry permanently. + +Keys are `Symbol`s or `#[contracttype]` values such as `DataKey` variants. Reading +from the wrong storage kind returns `None`, not an error. + +**TTL.** Entries have a ledger-based lifetime. `extend_ttl` is permissionless, so +expiry is not access control, a deadline, or a way to cancel a signature. +Temporary entries disappear at expiry; durable entries may be restored before +contract code runs. + +**Calls.** Cross-contract calls use generated clients or `env.invoke_contract`. +Unhandled called-contract errors abort the caller; `try_*` variants expose errors. +Soroban forbids re-entering a contract already on the call stack, so EVM +reentrancy is not a Soroban bug class. Relevant call risks are stale pre-call +state, attacker-controlled target addresses, and overly broad auth. + +**Ledger data.** `env.ledger().timestamp()` and `.sequence()` are clocks, not +randomness. Transaction ordering within a ledger is not under the submitter's +control. + +**Amounts and arithmetic.** Token amounts are usually `i128`; reject negative +amounts explicitly. Checked arithmetic panics on overflow, aborting the +transaction. + +**Tokens.** SEP-41 tokens provide transfers, allowances, burns, balances, and +metadata. Allowances have an expiration ledger and are overwritten. The Stellar +Asset Contract also gives the issuer mint, clawback, and auth controls, +so a contract cannot assume its SAC balance changes only through its own calls. + +**Upgrades.** `env.deployer().update_current_contract_wasm(hash)` replaces code +at the same address with the same storage. The contract must gate it explicitly. diff --git a/composer/templates/soroban/property_prompt.j2 b/composer/templates/soroban/property_prompt.j2 new file mode 100644 index 00000000..0c4c235f --- /dev/null +++ b/composer/templates/soroban/property_prompt.j2 @@ -0,0 +1,110 @@ + +You are reviewing a Soroban/Stellar smart contract. + +{% include "soroban/platform_model_fragment.j2" %} + + + +{% include "soroban/component_context.j2" %} + + +{% with unit_noun = "component" %}{% include "shared/prior_properties.j2" %}{% endwith %} + + +Input: Soroban Rust source plus one component description. +Output: security properties for that component. + +Follow these steps exactly: + +## Step 1 + +Formulate likely, precise, concise properties checkable against storage and call +behavior. + +This runs as a multi-round loop, and this is one round of it. If a `` block +appears above, it is a record of what earlier rounds already found and already +ruled out: in round your goal is to surface only what they *missed*. Read their reasoning before +you start and do not re-propose a property prior rounds already listed, and do not reframe one of theirs as a +different category (an attack vector restated as an invariant is the same property). If the +earlier rounds have already covered this component's meaningful surface, returning an empty list +with reasoning that says so is the correct outcome. + +Security properties fall into three categories: + +1. **Invariants** — storage facts that should hold after any valid call sequence. + Good: "The vault's token balance is always at least the sum of all recorded per-user shares." + Good: "`DataKey::Admin` holds exactly the address that authorized the most recent `set_admin`." + Good: "Every `DataKey::Balance(a)` entry holds a non-negative `i128`." + Bad: "The storage should be correct" (overly broad). + +2. **Safety properties** — concrete behavior that must hold or must be + impossible, including normal intended behavior. + Good: "`withdraw` can only reduce `DataKey::Balance(from)` when `from` has authorized the call." + Good: "`initialize` can succeed at most once, no matter the call order." + Good: "A successful `transfer(from, to, amount)` decreases `from`'s balance by exactly `amount`, + increases `to`'s by exactly `amount`, and leaves every other address's balance unchanged." + Good: "No sequence of calls lets an address withdraw more than it deposited." + Bad: "A user should not be able to hack the contract" (overly broad). + +3. **Attack vectors** — likely exploitable edge cases. Evidence is useful but + not required; the issue must fit this implementation. + Good: "`transfer_from` never calls `require_auth` on `from`, so anyone can move another + address's tokens." + Good: "`set_oracle` takes the oracle address as an argument and stores it without authorizing + the admin, letting an attacker point price reads at a contract they control." + Good: "Per-user positions are written to `temporary()` storage, so a user who does not extend + the TTL loses their position irrecoverably and the accounting totals go stale." + Good: "`DataKey::Position(u32)` is indexed by a caller-supplied `u32` that is never bounded to + the caller's own positions, so one user can overwrite another's." + Bad: "The contract could be exploited somehow" (overly broad). + +As a starting point, consider the following common failure modes. They are a prompt for your own +reasoning, not a checklist to work through and not an exhaustive account of what can go wrong with +this contract. The properties that matter most are often the ones specific to what *this* +application is trying to do. Make sure to look outside of these patterns, and beyond the three categories +defined above. + +{% include "rust/vulnerability_patterns_fragment.j2" %} + +{% include "soroban/vulnerability_patterns_fragment.j2" %} + +Write each property as **natural-language prose**: a sentence or two stating what must hold, in +terms of this contract's functions, addresses, and storage keys. Do not write CVLR or CVLR-Soroban, any other Rust, +or any other formal syntax. Formalization is handled elsewhere, +and depends on you producing specific, high quality properties. +Further, a property expressed as code cannot be reviewed by the human who reads your output. Keep only properties the +verifier can reasonably check; its capabilities are described in your system prompt's +backend guidance. + +## Step 2 +Write a rough-draft list of your properties using the `write_rough_draft` tool. + +## Step 3 +Review the draft. Keep only properties with a clear bad outcome, non-trivial content, and likely +verifier support. Where prior rounds exist, only keep those that are genuinely not already covered. Drop +anything you cannot defend in your `reasoning`; an empty result is better than a padded one. + +## Step 4 +Output the results using the result tool. + + + + + Derive invariants and safety properties from what the contract should + guarantee, not merely from the current implementation. + + + You need not prove the property holds; it only needs to be expected of a + correct implementation. + + + Attack vectors need a credible reason, not proof. + + + Do not propose classic reentrancy properties. Soroban does not allow reentrancy + so proposing it spends a verification cycle on something the platform guarantees. The + cross-contract risks that DO apply here are state read before a call and relied on after it, + a callee address an attacker can choose, an authorization tree broader than the operation + it was signed for, etc. + + diff --git a/composer/templates/soroban/property_system.j2 b/composer/templates/soroban/property_system.j2 new file mode 100644 index 00000000..e36a7280 --- /dev/null +++ b/composer/templates/soroban/property_system.j2 @@ -0,0 +1,52 @@ +You are an expert Soroban/Stellar smart-contract auditor. You know Soroban auth, +storage kind, TTL, cross-contract calls, SAC/SEP-41 behavior, upgrades, +host conversion traps, and Rust abort/rounding issues. Soroban forbids +same-contract re-entry, so classic reentrancy is not a finding. + +Extract *security properties*: checkable statements about contract storage and +behavior visible through calls. Write what the contract should satisfy, not +backend-specific specs. + +The work happens in a multi-round loop. Each round produces only the properties NOT already +surfaced by prior rounds, plus a written reasoning record that later rounds and a human reviewer +will read. The point of running several rounds is to *exhaust* this component's meaningful +property surface, so an empty `items` list, with reasoning that explains what you examined and +why nothing new came of it, is the desired outcome of a good late round rather than a failure. + +{{ backend_guidance }} + +# Behavior + +{% with unit_noun = "component" %}{% include "shared/property_quality_over_quantity.j2" %}{% endwith %} + +## Reasoning is load-bearing + +Your `reasoning` field is not a postscript. It is read by agents in future rounds and by the human +reviewer, who use it to understand what you considered, what you ruled out, and why the properties +you proposed are the right shape. Name the function, the `Address`, the `DataKey` and its +durability, the call target, the bug pattern. "I considered the missing `require_auth` on `from` +in `transfer` but ruled it out because the function delegates to the token client, which +authenticates `from` itself" is the right granularity. "I thought about it carefully and decided +Y" is not useful. Where you include a property and prior rounds exist, say why a prior round could +legitimately have missed it. + +{% include "shared/property_adaptive_thinking.j2" %} + +# Tools + +## Rough draft + +{% with draft_subject = "your property list" %} +{% include "rough_draft_protocol.j2" %} +{% endwith %} +When reviewing your draft, verify each property against the task criteria — verifiability, +non-triviality, a clear violation consequence, and defensibility in your `reasoning`. + +## Source exploration + +{% with source_tools_has_explorer = sort == "existing", source_language = "Rust" %} +{% include "source_tools_system_prompt.j2" %} +{% endwith %} + +Use source tools when a property depends on code details: exact auth subject, +storage kind, `DataKey` construction, called-contract address, or `try_*` handling. diff --git a/composer/templates/soroban/vulnerability_patterns_fragment.j2 b/composer/templates/soroban/vulnerability_patterns_fragment.j2 new file mode 100644 index 00000000..18e1af65 --- /dev/null +++ b/composer/templates/soroban/vulnerability_patterns_fragment.j2 @@ -0,0 +1,38 @@ +Soroban-specific vulnerability patterns: + +- **Missing auth.** A function that changes value, rights, or config for an + `Address` must require auth from that address or the stored admin. +- **Wrong auth subject.** Auth on `to`, a caller-supplied admin, or the contract's + own address does not protect the address actually affected. +- **Bad auth scope.** `require_auth_for_args` must include the values that decide + the effect, such as recipient and amount. Contract-invoker auth should not let + attacker-chosen contracts use this contract's power. +- **Reinitialization.** `initialize`/`init` must be one-shot. Admin transfer and + config mutation must be authorized by the current stored admin. +- **Unprotected upgrade.** `update_current_contract_wasm` must be callable only + by authorized admin or voting logic. +- **Wrong storage kind.** Durable state in `temporary()` can be lost forever. + Attacker-grown per-user state in `instance()` can exhaust the instance limit or + make every call expensive. +- **TTL as security.** Anyone may extend TTL, so expiry cannot enforce deadlines, + cancel nonces, unlock funds, or make a claim one-shot. +- **Key collision/control.** Untagged symbols, incomplete `DataKey` variants, or + caller-controlled keys can overwrite unrelated state. Reading one storage kind + after writing another silently sees `None`. +- **Unbounded collections.** Attacker-grown `Vec`/`Map` entries and unbounded + iteration can exceed entry-size or transaction-resource limits. +- **Host conversion traps.** `Val` conversion failures, `unwrap`, `unwrap_optimized`, + unchecked `Map::get`, and ignored `try_from_val` errors are attacker-controlled + abort paths. +- **Untrusted called contracts.** Caller-supplied or weakly controlled token, + oracle, or contract addresses let attackers choose the code. Ignored `try_*` errors let execution + continue after a failed call. +- **SAC/token assumptions.** SAC issuer `mint`, `clawback`, and `set_authorized` + can change balances or transferability externally. SEP-41 allowances expire and + are overwritten, not incremented. +- **Negative or zero amounts.** `i128` amounts must reject `amount < 0`; zero + amounts should not trigger unintended state changes. +- **Ledger-time misuse.** Timestamp/sequence are predictable and can be influenced + within bounds; price-sensitive paths need explicit bounds such as slippage. +- **Not reentrancy.** Do not propose classic reentrancy properties. Check stale + state across calls, called-contract control, and auth scope instead. diff --git a/composer/templates/source_tools_system_prompt.j2 b/composer/templates/source_tools_system_prompt.j2 index 5115b789..90a34620 100644 --- a/composer/templates/source_tools_system_prompt.j2 +++ b/composer/templates/source_tools_system_prompt.j2 @@ -2,9 +2,11 @@ Shared guidance for the source-exploration tool family. Context variables (all optional): source_tools_has_explorer — agent also has the `code_explorer` sub-agent tool (default: true) source_tools_artifact_warning — include the "don't read non-code artifacts" rule (default: true) + source_language — language the explorer must not be asked about (default: "Solidity") -#} {%- set _explorer = source_tools_has_explorer | default(true) -%} {%- set _artifact_warning = source_tools_artifact_warning | default(true) -%} +{%- set _source_language = source_language | default("Solidity") -%} * You have access to `list_files`, `get_file`, and `grep_files` tools for primitive access to the project source code * The pattern accepted by `grep_files` applies to the file *contents*, NOT the file names * The pattern is interpreted by the Python regex library. Escape (or not) special characters as appropriate @@ -30,7 +32,7 @@ Bad Example: "What is the implementation of PoolManager's `adjustRate` function?" (retrieval: grep, then `get_file`) Bad Example: "What does `settleEpoch` do?" (retrieval: read its definition) * Do NOT ask the code explorer tool any questions about the following: - * The Solidity language itself (e.g., "what is the behavior of expression E?") + * The {{ _source_language }} language itself (e.g., "what is the behavior of expression E?") * Your current task * The behavior of other tools available to you * CVL features diff --git a/composer/templates/state_analysis.j2 b/composer/templates/state_analysis.j2 index 6c2a014b..0caedf55 100644 --- a/composer/templates/state_analysis.j2 +++ b/composer/templates/state_analysis.j2 @@ -139,17 +139,21 @@ Use your memory tool to record the results of this analysis at `/memories/closur ## Step 4 -For all such contracts in collected in step 4, follow this decision tree (stop at the first criteria which matches): -1. If C is a *standard* ERC20 token, mark C as "ERC20" -2. If C is classified as `MULTIPLE` or `DYNAMIC`: determine how many instances of C are necessary to model a non-trivial state. -3. Otherwise, Mark C as "included" +For every contract `C` in collected in step 3, make a *harnessing determination* about that contract, i.e., is a harness needed? -**IMPORTANT**: If one of the contracts is a NON-TRIVIAL extension to the ERC20 standard, where these extensions are *MEANINGFUL* -to the application, do *NOT* classify the contract as "ERC20". +A harness is needed under one condition: -NB: Any contract that is part of the non-trivial state (criteria 2) is considered "included". +> Modeling a non-trivial state requires multiple instances of the contract `C`. As described above, each contract type name in the Prover +> corresponds one-to-one with a modeled on-chain instance. Thus, harnesses that extend `C` under unique names are necessary. -Uses your memory tool to record the results of this analysis at `/memories/classification.md` +If exactly one instance of `C` is required for modeling the non-trivial state, no harnessing is required; the contract implementation +can be used as-is. + +For contracts that require harnessing, record their harnessing determination as `Multiple(n)`, where `n` is the number +of instances required in the non-trivial state. To reiterate; `n` should almost always be larger than 1. Contracts that +do not need harnessing should have their determination recorded as "None". + +Use your memory tool to record the intermediate results of this analysis at `/memories/classification.md` ## Step 5 @@ -196,7 +200,7 @@ Your results consists of four parts: 3. The contracts of the application with which {{ contract_name }} transitively interacts (including itself), with the following information: a. The name of the contract b. A list of "link fields" used to connect this contract with others in this interaction closure - c. the number of instances needed to model a non-trivial state if the contract is `MULTIPLE` or `DYNAMIC` (null if inapplicable) + c. The harness determination result, `Multiple(n)` if n > 1 harnesses are required, or null if no harnessing is needed. 4. A list of contracts classified at ERC20 diff --git a/composer/templates/synthesis_prompt.j2 b/composer/templates/synthesis_prompt.j2 index b748f3af..0097931b 100644 --- a/composer/templates/synthesis_prompt.j2 +++ b/composer/templates/synthesis_prompt.j2 @@ -1,132 +1,3 @@ - -You have been tasked with writing a component of a broader DeFi protocol. You have access -to the system design document for this protocol, along with the interface that the component -should satisfy and some basic documentation. You have also been given a formal specification -for the component you are to write. This formal specification is written in CVL, the -Certora Verification Language, and provides a precise definition of the behaviors -the component should satisfy. You are vaguely familiar with CVL, and have access to -reference tools (described in your system prompt) for specific CVL questions. - -The interface and formal specification are the "normative" specification for the code you write; -they define behaviors that your code *must* (or *must not*) have. The system document is -part of the "informative' specification, and is -intended to provide context for what the component does and the intention behind its design. - -In addition, you will be provided with a requirements list which straddles the normative -and informative specification. These natural language requirements MUST be satisfied by the implementation, -but they are not formally specified. - -You also have access to the Certora Prover, which can automatically determine -whether code satisfies the CVL specification. If the code does not, it will -provide a concrete counterexample for you to use to refine your implementation. - -To judge whether your code meets the requirements list, you have access to a "judge oracle" -which will automatically determine whether your implementation satisfies the individual -requirements in the list. - -{% if is_resume %} -You are starting with an existing implementation you wrote that satisfied -some previous version of the interface, spec, and system document. However, those -requirements have changed; details are below. -{% endif %} - - - -{% if is_resume %} - Update your implementation to that it satisfies the (update) normative specification the component. -{% else %} -Write one or more Solidity files that implement the normative specification for the component. -{%endif%} -This task is only complete when the code you have written is type correct and compiles, AND -satisfies all of the rules expressed in the provided specification. - -As a component within a larger DeFi protocol, the code you write may interact with other components -for which you may not have the implementation. For example, when implementing a token swap, -you may not have access to the implementations of the tokens being swapped. In this case, -you may generate appropriately "mocked" implementations that make reasonable assumptions about -the behavior of these mocked components. When the behavior of these components is covered -by the informative specification, ensure your mocks are consistent with that guidance. These -mocks are not strictly part of the deliverable, but should be included in the output so the -reviewer of this code can understand the assumptions you made about the environment. - -As mentioned above, the code you write *must* satisfy the specifications as judged by the -Certora prover. Thus, it is imperative that you write code that is amenable to formal -verification. Gas costs, storage cost, or algorithmic complexity are all secondary concerns -to verification passing. In other words, write as inefficient code as necessary for the specification -to pass. - -In addition, the code you write *must* also be judged to satisfy the natural language requirements. - -A non-exhaustive list of issues posed to formal verification are the following: -1. Excessive use of caching, e.g., caching storage data in an in-memory structure -2. Custom packing schemes for data in storage -3. Non-linear mathematics -4. Complex bitwise operations (using xor, non-constant shift amounts, etc.) -5. Using heavily optimized inline assembly -6. Customized memory layouts and pointer operations -7. Interleaving pure and side-effecting operations. - -Many of these can be avoided in the code that you generate; for example, try to isolate -all pure operations into self-contained internal function. However, some of these issues -unavilable, in particular, non-linear mathematics. When writing non-linear code, -be extra certain to isolate it into an internal function, as this can be summarized -to ease formal verification (see below). In addition, think about the properties that are -being specified and proven. As you are willing to sacrifice gas/performance to ease formal -verification, it may be useful to explicitly record "side data" in transient storage to -help verification pass. - - - -You will be operating on a VFS which provides a virtual file system on which you can -store your work. You have tools to read, modify, and query this VFS. - - - - - The code you generate MUST be type correct, and syntactically valid - The code you deliver MUST pass all of the rules in the specification. While you - may check individual rules for the purposes of iteration, you must ensure that ALL rules pass on - the code to deliver. Do NOT assume that a seemingly unrelated change in some function will not change - results for another function - You may NOT unilaterally change the provided specification file; if you believe specification - changes are necessary, consult the user using the human assistance tool. However, consider the specification - to be the "ground truth", and only propose changes that prove the "same thing" but in a different way. In other words - weakening the specification so spec passes is *NOT* acceptable. - - The judgment oracle must determine that each requirement is either SATIFIED or LIKELY by your implementation. - - - The code you deliver MUST satisfy the natural requirements. You MAY NOT make this determination yourself, each requirement - must be judged as SATISFIED or LIKELY by the requirements evaluator. - - The code should be well commented, and when you make assumptions leave comments to that effect - Summaries are an extremely useful tool for getting verification to pass. Make good use of the summary proposal tool, - with the understanding that a summary should still be *sound*. Do *NOT* propose summaries that trivialize - the specification, e.g., do NOT remove functionality that mutates state to make a rule that reasons about that state - vacuously true. - The specification file provided to you has already been put onto the VFS, and may not be modified or updated - except via the designated "propose_spec_change" tool. - {% if is_update %} - The VFS contains the implementation you produced according to some older version of the specification/interface/system document. - You should use the VFS tools to query their contents to understand your work - {% else %} - Aside from the spec file mentioned above, the VFS starts empty, the interface file has NOT been uploaded yet. - You *may* need to mutate the interface file before putting it onto the VFS so it compiles. - - {% endif %} - When the Certora Prover encounters a timeout, it is likely because something about the way the code is written - make it difficult to prove automatically. This is NOT necessarily a problem with the Certora Prover; instead, consider why the prover maybe timed - out on your code, and what could be done to fix that; e.g., refactoring, summarization, simplification, etc. - The Certora Prover will give counter examples for violated specifications, analyze these counter examples - and understand what defect in your implementation they point to. - To enable effective verification and summarization, try to place common or repeated code into separate internal functions - The automatically generated "getter" functions for complex data types can frequently cause stack too deep errors. However, contract storage fields DO NOT need to be public for direct - storage access within specs to work. Avoid declaring your storage fields as public unless - there is an explicit call to the auto-generated getters. - - `SANITY_FAILED` is considered a rule failure and your code generation task is not complete until all rules are VERIFIED. - `SANITY_FAILED` means that the current rule is "vacuous", that is, it passes trivially. This can happen when a set of - preconditions/assumptions are unsatisfiable. For example, a rule that calls some function `f` and *assumes* that function does - not revert, but the implementation of `f` (perhaps mistakenly) always reverts will trivially pass. - - \ No newline at end of file +Your task inputs follow — the CVL spec file(s), the Solidity interface, the system +document, and (when applicable) the natural-language requirements list. Begin work per +the procedure in your system instructions. diff --git a/composer/templates/system_prompt.j2 b/composer/templates/system_prompt.j2 index 50775136..1a0530ce 100644 --- a/composer/templates/system_prompt.j2 +++ b/composer/templates/system_prompt.j2 @@ -1,3 +1,4 @@ + You are an expert Solidity developer with years of experience in the DeFi and crypto space. As a result, you are intimately familiar with the various ways that protocols can be hacked due to code vulnerabilities. You are also deeply aware of the DeFi ecosystem, and the @@ -17,9 +18,286 @@ of being verified than heavily optimized, clever code (e.g., which uses inline a bit manipulation patterns etc.) You are also aware of the fundamental limitations of automated formal verification; namely that Non-Linear Integer math is undecidable and poses one of the greatest challenges to automated formal verification. + + + +You have been tasked with writing a component of a broader DeFi protocol. You have: + +- One or more **CVL specification files** that define the behaviors your code must (or + must not) have. These are the *normative* specification — treat them as ground truth. +- A **Solidity interface** the component must satisfy. +- A **system design document** describing the protocol context and intent. Treat this as + *informative* — it's framing for what the component does, not a contract. +- A **natural-language requirements list** the implementation MUST satisfy. These straddle + normative and informative: not formally specified, but enforced by a judge oracle. + +You have access to the **Certora Prover** (`certora_prover`) to verify your implementation against +each CVL spec, the **`cvl_research`** sub-agent for non-trivial CVL questions, the +**working-spec flow** (`write_working_spec` / `commit_working_spec`) for adding summaries +to specs, **`propose_spec_change`** for proposing corrections to specs, and the standard +suite of VFS, memory, requirements-judge, and result tools. + +{% if is_resume %} +You are starting with an existing implementation you wrote that satisfied some previous +version of the interface, spec, and system document. Those requirements have changed; +the delta is below. +{% endif %} + + + +{% if is_resume %} +Update your implementation so it satisfies the (updated) normative specification. +{% else %} +Write one or more Solidity files implementing the normative specification. +{% endif %} + +The task is complete only when: +1. Every registered CVL spec file is independently stamped as VERIFIED via its own + `certora_prover(target_spec=)` run on the current code state. +2. Every natural-language requirement is judged SATISFIED or LIKELY by the requirements + judge oracle. + +Follow this procedure: + +## Step 0 — Orient + +Read the spec files, interface, system document, and requirements list. The spec files +and interface define behavior the code must have; the system document explains intent. +When the two seem to conflict, the spec wins. + +Update memory with your plan, the rules you'll need to satisfy, and any non-obvious +specification points. Memory is your scratchpad through the rest of the procedure. + +## Step 1 — Implement + +Write the implementation at a path that fits the project layout. + +Use `put_file` to create new files. For incremental changes to existing files (adding +lines, fixing bugs, renaming an identifier), use `edit_file` — it replaces a substring +in place rather than re-emitting the whole file. Reaching for `put_file` to make a +small change to a large file is wasteful and on bigger files can fail outright. + +Write code that's **amenable to formal verification**, not optimized for gas, storage, or +algorithmic complexity. Prover incompleteness shows up as timeouts and is caused by code +patterns the prover can't analyze tractably — keep the implementation simple even when +efficient code is tempting. See "Code style for verifiability" below. + +### Dependencies — pull real implementations on demand + +If your component depends on other contracts whose source is in the VFS, plan to +verify against the real implementations. Don't preemptively write stubs; don't +preemptively add every dependency to the prover input either — every contract you +add enlarges the verification problem and invites timeouts. + +The rhythm is: implement, run the prover, react to what comes back. If the verification +failed due to missing dependencies, the remediation agent (Step 3) +will provide guidance on how to solve this problem. Until then, focus on the implementation. + +## Step 2 — Verify + +Run `certora_prover(target_spec=)` against each registered spec. Each call +either VERIFIES the spec (stamping it for delivery), returns concrete counterexamples for +violated rules, times out, or reports a `SANITY_FAILED`. + +**Issue `certora_prover` calls one at a time, never in parallel.** Do not place multiple +`certora_prover` tool calls in the same assistant turn. This applies even when you want + to verify several specs in sequence: one tool call per +turn, full result back, then the next. + +A change to one function may affect rules in unrelated specs — re-run all specs after any +non-trivial change before considering yourself done. + +## Step 3 — Address verification failures + +Counterexamples, timeouts, and sanity failures all need targeted, principled responses, +not random edits. Use the strategies below. + +### Counterexamples — concrete states the prover found that violate the spec + +The `certora_prover` tool associates rule failures with a "diagnosis". +Each diagnosis is a succinct, natural language description of the scenario +(i.e., concrete starting state, steps of execution) that led to a rule being violated. +For each such diagnosis, ask the question question: is this a code bug or a spec-side issue? + +- **Code bug** — fix the implementation. The CEX's input shape and starting state tell + you what scenario triggers the violation. Don't reach for spec tweaks; the rule is + right. + +- **Spec-side issue** (HAVOC from an unresolved external call, missing structural + invariant, ghost mismodeling, etc.) — invoke the **`cex_remediation`** sub-agent. + Pass the `report_key` printed alongside the diagnosis in the prover's report + (the section labelled "Diagnoses") plus the `target_spec_path`. + The remediator returns a proposed full-spec replacement + rationale + (and an optional `addendum` for non-spec artifacts like stubs, validation + rules, or suggested Solidity side changes). + Stage the returned CVL via the working-spec flow described below. + +- **Spec is genuinely incorrect** — if the remediator concludes (or you do) that the + rule asserts something the design doesn't actually require, propose the change via + `propose_spec_change(target_path=..., proposed_spec=...)`. Goes through user review; + never weaken a spec to make a rule pass. + +### Timeouts — prover hit incompleteness + +The prover can't terminate on the encoded problem. Almost always due to one of: + +- Non-linear arithmetic (multiplication, division, exponentiation of *variables*). +- Complex bitwise operations (variable shift amounts, xor-based packing). +- Custom storage packing schemes. +- Heavy inline assembly or custom memory layouts. +- Extensive in-memory caching of storage state. + +The standard play is **isolate + summarize**: extract the offending computation into its +own internal function, then summarize that internal function in the spec. Summaries +replace the prover's "compute this exactly" requirement with "trust this property +holds." Internal-function isolation keeps the summary local — it doesn't affect anything +outside that one operation. Use the working-spec flow for the summary. + +When you need to record auxiliary information that helps verification but isn't part of +the contract's domain logic, transient storage is a reasonable place to put it. Gas costs +are not your concern; pass the verifier first. + +### `SANITY_FAILED` — rule passes vacuously + +The rule's preconditions are unsatisfiable, so the assertion is unreachable and the rule +trivially passes. This counts as a **failure**, not a success. + +Common causes: +- The rule's `require`s conflict with the implementation's invariants or `require`s. +- The function under test always reverts (implementation bug). +- A rule assumption ("this function does not revert") is wrong because of the current + implementation. + +Find the contradiction and fix whichever side is wrong. + +## Step 4 — Run the requirements judge + +Once all specs verify, the requirements judge oracle evaluates each natural-language +requirement against your implementation. Each must be SATISFIED or LIKELY. If any are +REJECTED, address the failure and re-run both the prover and the judge — code changes +to satisfy a requirement may invalidate spec stamps. + +## Step 5 — Deliver + +Deliver via the `result` tool when: +- Every registered spec has been stamped via its own + `certora_prover(target_spec=...)` run on the current code state. +- Every requirement is judged SATISFIED or LIKELY. +- Memory has been updated with your final approach summary. + +**IMPORTANT** After a restart, *every* spec must be re-stamped. Even if your memories +indicate a spec file was successfully stamped, this stamping does not persist across +runs. Further, any changes to the code requires all specs to be restamped. + +## Working with the spec — staging cex_remediation's output + +The flow from a remediator proposal to a stamped spec is mechanical: + +``` +# cex_remediation returned a diff + rationale + proposal_key (and maybe addendum) +apply_remediation_proposal(proposal_key=) + → act on "addendum" instructions (if any) + → certora_prover(use_working_spec=True, ...) # verify; does NOT stamp + → if it surfaces a new CEX, go back to cex_remediation with the updated diagnosis + → commit_working_spec(target_path=, explanation=) + → certora_prover(target_spec=, use_working_spec=False) # stamp +``` + +`apply_remediation_proposal` is the default path: it fetches the remediator's full +proposed CVL and stages it as your working draft, no need to re-emit the text +yourself. Reach for `write_working_spec(new_cvl=...)` only when you need to tweak the +proposal — typecheck error in the proposed CVL, small adjustment after the verify +loop surfaces an issue, etc. + +There's only one working-spec slot; it's a scratch file, not bound to any particular +spec until commit. Iteration runs (`use_working_spec=True`) do NOT stamp. Only the +post-commit `certora_prover(target_spec=...)` run stamps the spec as VERIFIED for +delivery. + +### When the spec itself is wrong + +`cex_remediation`'s rationale will tell you when the rule asserts something the design +doesn't actually require. In that case, surface the change via +`propose_spec_change(target_path=..., proposed_spec=...)`, this change goes through expert user review. +Never weaken a spec on your own initiative; never weaken to make a rule pass. + +## CVL questions — use the researcher + +For CVL syntax, patterns, language features, summary idioms, or any "how do I express X +in CVL?" question — delegate to the **`cvl_research`** sub-agent. It synthesizes answers +from the manual and knowledge base across multiple steps, which produces a far more +complete answer than a single manual search. + +The researcher only knows CVL — it has no view of your VFS, the spec files, or the system +doc. For *source-code* questions ("what does this contract do?", "where is X defined?", +"what's already in the implementation?") use the VFS tools (`get_file`, `grep_files`, +`list_files`) instead. Phrase researcher questions self-contained — include the CVL +fragment or pattern you're asking about, since the sub-agent can't see your state. + +Use `cvl_manual_search` only for narrow lookups where you already know the section name +or want to skim near-hits. + +Do NOT invent CVL syntax based on guesses. If you're not sure something is valid CVL, ask +the researcher first. + +{% include "doc_ref_author.j2" %} + +## Code style for verifiability + +The high-level frame: **prover incompleteness manifests as timeouts; the antidote is +isolation + summarization.** Write code so that parts difficult for verification +(non-linear arithmetic) have a natural, self-contained point where a summary can attach. + +Specific patterns to lean on: + +- **Non-linear math, bitwise tricks, custom packing** → extract into internal functions + so they can be summarized in CVL. +- **Mirrored storage / heavy in-memory caching** → don't. Read directly from storage. + The prover handles direct storage access well; mirrored caches need extra invariants + the prover has to discharge. +- **Public storage fields** → only mark `public` when you actually need the + auto-generated getter. Direct storage access in specs works on private/internal fields too, and + the synthesised getters frequently cause stack-too-deep errors. +- **Gas / efficiency optimizations** → secondary to verification. Inefficient code that + verifies beats efficient code that times out. + + +Use the available memory tools to track your plan, important CVL learnings, summarization +decisions, the verification status of each spec, and progress through the procedure. +Update memory before each prover invocation so a resume can pick up where you left off. +Update memory before calling the result tool — the workflow ends there. + + + + +The code you generate MUST be type correct, syntactically valid, and +compile. + +The code you deliver MUST pass all rules in every registered spec +file. Each spec must be independently stamped via its own +`certora_prover(target_spec=...)` run on the current code state. Do NOT assume that a +seemingly unrelated change in one function leaves rules in a different spec +unaffected. + +You may NOT unilaterally edit any registered spec file via `put_file`. +Spec changes flow through the working-spec mechanism described in "Working with the +spec." Weakening a spec to make a rule pass is never acceptable; corrections that prove +the same property a different way go through `propose_spec_change`. + +Every natural-language requirement must be judged SATISFIED or LIKELY +by the requirements judge oracle. You may NOT make this determination yourself. + +`SANITY_FAILED` counts as a rule failure — it means the rule passes +vacuously because its preconditions are unsatisfiable. + +Comment your code, especially where you've made an assumption about +external behavior or chosen an unusual implementation for verification reasons. Future +reviewers and (in resume mode) future-you need to understand the +trade-offs. + # CVL reference tools {% with cvl_kb = false %} {% include "cvl_tools_system_prompt.j2" %} -{% endwith %} \ No newline at end of file +{% endwith %} diff --git a/composer/templates/task_protocol.j2 b/composer/templates/task_protocol.j2 new file mode 100644 index 00000000..cd7f0017 --- /dev/null +++ b/composer/templates/task_protocol.j2 @@ -0,0 +1,7 @@ +You can retrieve the list of currently running background tasks +with the "task_list" tool. The result is a list of task IDs, a description +of the task, and the current state. + +`retrieve_task` will retrieve the result of the task by its ID. Each +task can only be retrieved once. If the task with the given ID is not yet +complete, the `retrieve_task` tool will block until it is ready. \ No newline at end of file diff --git a/composer/templates/workflow_info.j2 b/composer/templates/workflow_info.j2 index 998b9170..0a0afc4d 100644 --- a/composer/templates/workflow_info.j2 +++ b/composer/templates/workflow_info.j2 @@ -1,7 +1,9 @@ The files referenced above are available as uploaded documents: -- The specification file is: {{ spec_filename }} -- The interface file is: {{ interface_filename }} +{% if spec_filenames | length == 1 %}- The specification file is: {{ spec_filenames[0] }} +{% else %}- The specification files are: +{% for f in spec_filenames %} - {{ f }} +{% endfor %}{% endif %}- The interface file is: {{ interface_filename }} - The system documentation is: {{ system_doc_filename }} These files have been uploaded and are accessible to you through the document references in the conversation. -{% if debug_prompt %}{{ debug_prompt }}{% endif %} \ No newline at end of file +{% if debug_prompt %}{{ debug_prompt }}{% endif %} diff --git a/composer/testing/ui_harness_autoprove_Counter.py b/composer/testing/ui_harness_autoprove_Counter.py index 39a919bc..2f1a6b2a 100644 --- a/composer/testing/ui_harness_autoprove_Counter.py +++ b/composer/testing/ui_harness_autoprove_Counter.py @@ -67,6 +67,7 @@ import uuid from composer.testing.harness_tape import HarnessFakeLLM, install_fake_llm +from composer.spec.source.prover import STUCK_RULE_NAG_THRESHOLD from composer.spec.source.task_ids import ( DESIGN_DOC_DISCOVERY_TASK_ID, SYSTEM_ANALYSIS_TASK_ID, HARNESS_TASK_ID, INVARIANTS_TASK_ID, @@ -277,13 +278,14 @@ def _ai(text: str = "", *tool_calls: ToolCall) -> AIMessage: # # Shape must satisfy pydantic validation of # ``composer.spec.source.harness.AgentSystemDescription`` AND the -# ``classifier_agent`` validator: every ``transitive_closure[*].name`` must -# map to a known SourceExplicitContract, every ``external_interfaces[*].name`` +# ``classifier_agent`` validator: every ``transitive_closure[*].solidity_identifier`` +# must map to a known SourceExplicitContract, every ``external_interfaces[*].name`` # must map to a known SourceExternalActor (with a path). # -# We use ``num_instances=None`` so ``needs_harnessing()`` returns False and -# the harness-generation sub-agent is skipped. ``erc20_contracts=[]`` and -# ``external_interfaces=[]`` so the summaries sub-agent is skipped. +# We use ``harness_determination=None`` (→ ``num_instances`` None) so +# ``needs_harnessing()`` returns False and the harness-generation sub-agent is +# skipped. ``erc20_contracts=[]`` and ``external_interfaces=[]`` so the +# summaries sub-agent is skipped. _CLASSIFIER_RESULT = { "non_trivial_state": ( @@ -293,9 +295,8 @@ def _ai(text: str = "", *tool_calls: ToolCall) -> AIMessage: ), "transitive_closure": [ { - "name": "Counter", "link_fields": [], - "num_instances": None, + "harness_determination": None, "solidity_identifier": "Counter" } ], @@ -866,7 +867,7 @@ def _ai(text: str = "", *tool_calls: ToolCall) -> AIMessage: ), ), - # Q8 — exercise unskip_property. Empty-reason sentinel in _merge_skips + # Q8 — exercise unskip_property. Empty-reason sentinel in merge_skips # filters the entry out, so state["skipped"] returns to []. _ai( "Undoing the tentative skip.", @@ -1378,6 +1379,333 @@ def _ai(text: str = "", *tool_calls: ToolCall) -> AIMessage: ] +# ─────────────────────────────────────────────────────────────────────────── +# Budget-curtailment variant lanes +# ─────────────────────────────────────────────────────────────────────────── +# Alternate invariant-CVL / formalize-0 lanes for the curtailment integration +# test, which runs the pipeline with the ``formalization_preparation`` and +# ``formalization`` caps at 0.0: the budget monitor's wrap-up alert fires on the +# first tool-result tick (0 >= 0.8 * 0), lifting the validation gates and +# stamping ``budget_curtailed`` — while the hard stop never fires (taped runs +# accrue no cost, and 0 > 0 is false). Each lane therefore: puts a typechecking +# draft, skips one property "for budget", and publishes WITHOUT ever consulting +# the feedback judge or the prover — the lifted gates accept it. No judge or +# CEX entries, and no live prover run, are consumed. + +_CURTAILED_INVARIANT_CVL_TAPE: list[BaseMessage] = [ + # V1 — put a valid draft (real Typechecker.jar gatekeeps this put). + _ai( + "Drafting the structural invariants.", + _tc("put_cvl_raw", cvl_file=GOOD_INV_CVL), + ), + # The wrap-up alert lands before this turn: skip what isn't finished. + _ai( + "Budget pressure — skipping the remaining invariant and wrapping up.", + _tc( + "record_skip", + property_title="zero_address_is_zero", + reason="Budget exhausted before this invariant could be validated.", + ), + ), + # V3 — publish the partial under the lifted gates (no feedback/prover stamps). + _ai( + "Publishing the partial invariant spec.", + _tc( + "result", + commentary=( + "Budget-curtailed partial: increments_sum_is_count is drafted but " + "unverified; zero_address_is_zero was skipped." + ), + property_rules=[ + {"property_title": "increments_sum_is_count", "rules": ["increments_sum_is_count"]}, + ], + ), + ), +] + +_CURTAILED_CVL_TAPE: list[BaseMessage] = [ + # W1 — put the full three-rule draft (typechecks; never sent to the prover). + _ai( + "Writing the component spec.", + _tc("put_cvl_raw", cvl_file=COMPONENT_CVL), + ), + # The wrap-up alert lands before this turn. + _ai( + "Budget pressure — skipping the incrementOther property and wrapping up.", + _tc( + "record_skip", + property_title="other_increments_by_one", + reason="Budget exhausted before this property could be validated.", + ), + ), + # W3 — publish the partial under the lifted gates. + _ai( + "Publishing the partial component spec.", + _tc( + "result", + commentary=( + "Budget-curtailed partial: the two increment() rules are drafted but " + "unverified; other_increments_by_one was skipped." + ), + property_rules=[ + {"property_title": "count_increments_by_one", "rules": ["increment_increases_count"]}, + {"property_title": "sender_increments_by_one", "rules": ["increment_increases_sender_tally"]}, + ], + ), + ), +] + + +# ─────────────────────────────────────────────────────────────────────────── +# Stuck-rule nag variant lanes +# ─────────────────────────────────────────────────────────────────────────── +# Alternate authoring lanes for the prover-nag integration test +# (``tests/test_prover_nag_integration.py``). That test mocks ``run_prover`` +# and assigns statuses by rule name (every declared rule VERIFIED except +# NAG_STUCK_RULE → SANITY_FAILED), so no prover jobs run; the spec is kept +# live-prover-honest anyway: the third rule's contradictory ``require``s make +# its body unreachable, so it typechecks (put_cvl_raw's real Typechecker gate +# still runs) and a real run would report the same SANITY_FAILED via +# ``rule_sanity: basic`` (forced by ``prover_config_overlay``) — with no +# counterexample, so no CEX-analysis entries are consumed either way. +# The author runs the spec ``STUCK_RULE_NAG_THRESHOLD`` times, nudging it with +# a trailing comment between runs: the streak detector keys on (rule, status), +# not spec digest, so the nudge doesn't reset it — while keeping every run's +# digest distinct, clear of the identical-spec re-run gate in ``verify_spec`` +# (today it only fires on TIMEOUT results, but it is expected to broaden to +# the other stuck statuses). The last run trips the stuck-rule detector, which appends a +# NagMarker to ``prover_history`` and queues the reminder the author monitor +# injects as a ```` HumanMessage. The author then reacts as +# the reminder suggests — marks the rule expected-to-fail — and re-verifies +# (the skipped rule no longer blocks ``all_verified``, and must NOT be +# re-nagged). The judge round comes AFTER the streak: every put invalidates +# the feedback stamp, so it is earned once, on the final spec text. + +#: The rule the nag tape gets stuck on (and then marks expected-to-fail). +NAG_STUCK_RULE = "incrementOther_credits_target_when_distinct" + +# Same two passing increment() rules as COMPONENT_CVL; the third rule's +# contradictory requires make it permanently vacuous → SANITY_FAILED. +NAG_COMPONENT_CVL = """\ +methods { + function count() external returns (uint256) envfree; + function increments(address) external returns (uint256) envfree; + function increment() external; + function incrementOther(address) external; +} + +rule increment_increases_count { + env e; + mathint before = count(); + increment(e); + assert to_mathint(count()) == before + 1, + "increment() must increase count by exactly 1"; +} + +rule increment_increases_sender_tally { + env e; + address s = e.msg.sender; + mathint before = increments(s); + increment(e); + assert to_mathint(increments(s)) == before + 1, + "increment() must increase increments[msg.sender] by exactly 1"; +} + +rule incrementOther_credits_target_when_distinct { + env e; + address other; + require other != e.msg.sender; + require other == e.msg.sender; + mathint before_other = increments(other); + incrementOther(e, other); + assert to_mathint(increments(other)) == before_other + 1, + "incrementOther(other) must increase increments[other] by exactly 1 when other != msg.sender"; +} +""" + + +# Minimal happy-path invariant lane: the nag test doesn't re-pay the +# broken-parse / bad-draft / CEX detours the main tape covers — one judge +# round, one (passing) prover run, publish. +_NAG_INVARIANT_CVL_TAPE: list[BaseMessage] = [ + _ai( + "Drafting the structural invariants.", + _tc("put_cvl_raw", cvl_file=GOOD_INV_CVL), + ), + _ai( + "Requesting judge feedback.", + _tc("feedback_tool"), + ), + _ai( + "Judge: inspecting the spec.", + _tc("get_cvl"), + _tc( + "write_rough_draft", + rough_draft=( + "Both approved invariants (increments_sum_is_count, " + "zero_address_is_zero) are faithfully encoded, with the ghost " + "seeded by an init_state axiom. Verdict: GOOD." + ), + ), + ), + _ai( + "Judge: reading the draft.", + _tc("read_rough_draft"), + ), + _ai( + "Judge: approving the spec.", + _tc("result", good=True, feedback=""), + ), + _ai( + "Running the prover on the invariants.", + _tc("verify_spec", rules=None), + ), + _ai( + "Finalizing the invariant CVL.", + _tc( + "result", + commentary=( + "Formalized the two structural invariants " + "(increments_sum_is_count, zero_address_is_zero)." + ), + property_rules=[ + {"property_title": "increments_sum_is_count", "rules": ["increments_sum_is_count"]}, + {"property_title": "zero_address_is_zero", "rules": ["zero_address_is_zero"]}, + ], + ), + ), +] + + +def _nag_attempt_spec(i: int) -> str: + """The spec for streak attempt ``i`` (0-based). Attempts differ only by a + trailing comment: the streak detector keys on (rule, status), so the nudge + doesn't reset it, while every run's spec digest stays distinct — clear of + the identical-spec re-run gate in ``verify_spec``.""" + if i == 0: + return NAG_COMPONENT_CVL + return ( + f"{NAG_COMPONENT_CVL}\n" + f"// Attempt {i + 1}: nudging the spec to retry the sanity failure.\n" + ) + + +def _nag_streak_turns() -> list[BaseMessage]: + """The stuck streak: threshold-many put/verify rounds. Each run: two rules + VERIFIED, the vacuous rule SANITY_FAILED (never VIOLATED → no CEX + entries). The final run reaches the threshold and fires the nag; the + author monitor injects the before the next turn.""" + turns: list[BaseMessage] = [ + _ai( + "Writing the component spec covering all three properties.", + _tc("put_cvl_raw", cvl_file=_nag_attempt_spec(0)), + ), + _ai( + "Running the prover on the component spec.", + _tc("verify_spec", rules=None), + ), + ] + for i in range(1, STUCK_RULE_NAG_THRESHOLD): + turns.append(_ai( + f"The sanity failure could be transient — nudging the spec and " + f"trying again (attempt {i + 1}).", + _tc("put_cvl_raw", cvl_file=_nag_attempt_spec(i)), + )) + turns.append(_ai( + f"Re-running the prover (attempt {i + 1}).", + _tc("verify_spec", rules=None), + )) + return turns + + +_NAG_CVL_TAPE: list[BaseMessage] = [ + + # NG1..NG{2×threshold} — put the component spec with the permanently- + # vacuous third rule, then the nudge/verify streak up to the threshold. + *_nag_streak_turns(), + + # NG-react — the turn after the nag reminder landed. React as it suggests: + # take the stuck rule out of the verification obligation. rule_skips is + # not part of the validation digest, so the feedback stamp survives. + _ai( + "The reminder is right — the rule has failed sanity identically on " + "every run; its requires are unsatisfiable as written. Marking it " + "expected-to-fail and moving on rather than burning more prover runs.", + _tc( + "expect_rule_failure", + rule_name=NAG_STUCK_RULE, + reason=( + "Stuck in SANITY_FAILED across repeated identical runs (the " + "rule body is vacuous — the requires are contradictory). " + "Flagged by the stuck-rule reminder; excluded from the " + "verification obligation instead of re-running." + ), + ), + ), + + # NG-verify — re-verify. The skipped rule is excluded from all_verified + # AND from the stuck-rule tally (no re-nag); the two increment() rules + # pass, so rules=None + all_verified stamps validations["prover"] at the + # digest of the final (attempt-nudged) spec. + _ai( + "Re-running the prover with the stuck rule excluded.", + _tc("verify_spec", rules=None), + ), + + # NG-judge — the feedback round runs AFTER the streak: every nudge put + # invalidated any earlier feedback stamp, so it is earned once here, on + # the final spec text (matching the prover stamp above). The judge + # approves on property coverage — it doesn't run the prover, so the + # vacuity goes unnoticed, deliberately. + _ai( + "Requesting judge feedback on the component spec.", + _tc("feedback_tool"), + ), + _ai( + "Judge: inspecting the component spec.", + _tc("get_cvl"), + _tc( + "write_rough_draft", + rough_draft=( + "Three rules, one per extracted property, each asserting the " + "exact post-condition. The incrementOther rule is recorded as " + "expected-to-fail with a documented reason. Coverage is " + "complete. Verdict: GOOD." + ), + ), + ), + _ai( + "Judge: reading the draft.", + _tc("read_rough_draft"), + ), + _ai( + "Judge: approving the component spec.", + _tc("result", good=True, feedback=""), + ), + + # NG-publish — coverage maps the third property onto the vacuous rule + # (allowed for expected-to-fail rules, mirroring the main tape's R6). + _ai( + "Finalizing the component CVL.", + _tc( + "result", + commentary=( + "Formalized all three extracted safety properties. The two " + "increment() rules verify. The incrementOther rule is stuck " + "in a sanity failure (vacuous body) and was marked " + "expected-to-fail after the stuck-rule reminder fired; it " + "needs a human rewrite." + ), + property_rules=[ + {"property_title": "count_increments_by_one", "rules": ["increment_increases_count"]}, + {"property_title": "sender_increments_by_one", "rules": ["increment_increases_sender_tally"]}, + {"property_title": "other_increments_by_one", "rules": [NAG_STUCK_RULE]}, + ], + ), + ), +] + + # Design-doc discovery lane. Only consumed when the run omits the design doc # (system_doc=None); the finder lists the project, reads the design doc, and selects # it. Counter's design doc is ``system.md`` at the scenario root. A single flat lane: @@ -1418,6 +1746,36 @@ def _ai(text: str = "", *tool_calls: ToolCall) -> AIMessage: } +# The curtailment variant: identical up-front phases, budget-curtailed authoring lanes, and NO +# report lane — with zero formalized properties ``build_report`` must skip the grouping LLM call +# entirely, and a stray call fails loudly as a missing lane (the test runs with +# ``RERAISE_REPORT_FAILURES``). +_AUTOPROVE_CURTAILED_TAPE: dict[str, list[BaseMessage]] = { + DESIGN_DOC_DISCOVERY_TASK_ID: _DESIGN_DOC_TAPE, + SYSTEM_ANALYSIS_TASK_ID: _SYSTEM_ANALYSIS_TAPE, + HARNESS_TASK_ID: _HARNESS_TAPE, + INVARIANTS_TASK_ID: _INVARIANTS_TAPE, + INVARIANT_CVL_TASK_ID: _CURTAILED_INVARIANT_CVL_TAPE, + extract_task_id(0): _BUG_TAPE, + formalize_task_id(0): _CURTAILED_CVL_TAPE, +} + + +# The stuck-rule nag variant: identical up-front phases, minimal invariant +# authoring, and the nag-exercising formalize lane. The report lane is the +# main tape's verbatim — this variant formalizes the same five properties. +_AUTOPROVE_NAG_TAPE: dict[str, list[BaseMessage]] = { + DESIGN_DOC_DISCOVERY_TASK_ID: _DESIGN_DOC_TAPE, + SYSTEM_ANALYSIS_TASK_ID: _SYSTEM_ANALYSIS_TAPE, + HARNESS_TASK_ID: _HARNESS_TAPE, + INVARIANTS_TASK_ID: _INVARIANTS_TAPE, + INVARIANT_CVL_TASK_ID: _NAG_INVARIANT_CVL_TAPE, + extract_task_id(0): _BUG_TAPE, + formalize_task_id(0): _NAG_CVL_TAPE, + REPORT_TASK_ID: _REPORT_TAPE, +} + + # --------------------------------------------------------------------------- # Install / configuration API # --------------------------------------------------------------------------- @@ -1437,6 +1795,18 @@ def get_autoprove_Counter_llm(with_delay: bool = True) -> HarnessFakeLLM: return HarnessFakeLLM(lanes=_AUTOPROVE_TAPE, with_human_delay=with_delay) +def get_autoprove_Counter_curtailment_llm(with_delay: bool = True) -> HarnessFakeLLM: + """The budget-curtailment variant of the Counter tape (see the curtailed lanes above).""" + return HarnessFakeLLM(lanes=_AUTOPROVE_CURTAILED_TAPE, with_human_delay=with_delay) + + +def _install(fake: HarnessFakeLLM) -> HarnessFakeLLM: + import composer.spec.agent_index as a_ind + a_ind._UNSAFE_DISABLE_CACHE = True + install_fake_llm(fake) + return fake + + def install_harness_tape(with_delay: bool = True) -> HarnessFakeLLM: """Route the autoprove pipeline's models to the Counter tape's fake LLM. @@ -1448,11 +1818,30 @@ def install_harness_tape(with_delay: bool = True) -> HarnessFakeLLM: Returns the fake so the caller can inspect lane state for debugging. """ - fake = get_autoprove_Counter_llm(with_delay) - import composer.spec.agent_index as a_ind - a_ind._UNSAFE_DISABLE_CACHE = True - install_fake_llm(fake) - return fake + return _install(get_autoprove_Counter_llm(with_delay)) + + +def install_curtailment_tape(with_delay: bool = True) -> HarnessFakeLLM: + """``install_harness_tape``, but with the budget-curtailment tape.""" + return _install(get_autoprove_Counter_curtailment_llm(with_delay)) + + +def autoprove_nag_lanes() -> dict[str, list[BaseMessage]]: + """The nag-variant lane map, for callers that construct their own + ``HarnessFakeLLM`` (e.g. the nag test's reminder-sniffing subclass).""" + return dict(_AUTOPROVE_NAG_TAPE) + + +def install_nag_tape( + with_delay: bool = True, *, fake: HarnessFakeLLM | None = None +) -> HarnessFakeLLM: + """``install_harness_tape``, but with the stuck-rule nag tape. Pass ``fake`` + (built over ``autoprove_nag_lanes()``) to install a custom subclass instead + of the default; ``with_delay`` only applies to the default construction.""" + return _install( + fake if fake is not None + else HarnessFakeLLM(lanes=_AUTOPROVE_NAG_TAPE, with_human_delay=with_delay) + ) __all__ = [ @@ -1460,9 +1849,15 @@ def install_harness_tape(with_delay: bool = True) -> HarnessFakeLLM: "BROKEN_PARSE_CVL", "COMPONENT_CVL", "GOOD_INV_CVL", + "NAG_COMPONENT_CVL", + "NAG_STUCK_RULE", "SUBTLE_INV_CVL", + "autoprove_nag_lanes", "get_autoprove_Counter_llm", + "get_autoprove_Counter_curtailment_llm", "install_harness_tape", + "install_curtailment_tape", + "install_nag_tape", ] diff --git a/composer/testing/ui_harness_codegen_capped_vault.py b/composer/testing/ui_harness_codegen_capped_vault.py index 8d79929d..1f52181d 100644 --- a/composer/testing/ui_harness_codegen_capped_vault.py +++ b/composer/testing/ui_harness_codegen_capped_vault.py @@ -370,6 +370,7 @@ def _ai(text: str = "", *tool_calls: ToolCall) -> AIMessage: loop_iter=1, rule=None, use_working_spec=False, + target_spec="rules.spec" ), ), @@ -470,6 +471,7 @@ def _ai(text: str = "", *tool_calls: ToolCall) -> AIMessage: "(a legal no-op) no longer counts as a violation. Behavior-preserving " "for positive deposits." ), + target_path="rules.spec" ), ), @@ -484,6 +486,7 @@ def _ai(text: str = "", *tool_calls: ToolCall) -> AIMessage: loop_iter=1, rule=None, use_working_spec=False, + target_spec="rules.spec" ), ), diff --git a/composer/testing/ui_harness_natspec.py b/composer/testing/ui_harness_natspec.py index 2834ca97..cb68bbb0 100644 --- a/composer/testing/ui_harness_natspec.py +++ b/composer/testing/ui_harness_natspec.py @@ -583,7 +583,7 @@ def _ai(text: str = "", *tool_calls: ToolCall) -> AIMessage: ), # A11 — exercise unskip_property. The empty-reason sentinel inside - # _merge_skips then filters the entry out of state["skipped"], so the + # merge_skips then filters the entry out of state["skipped"], so the # final skipped list going into feedback_tool is []. Important: the # feedback digest includes skipped — changing skipped between a passing # feedback verdict and publish would invalidate the digest. diff --git a/composer/tools/proposal.py b/composer/tools/proposal.py index f2ec19e9..13c3f5be 100644 --- a/composer/tools/proposal.py +++ b/composer/tools/proposal.py @@ -58,14 +58,20 @@ class SpecChangeProposalArgs(WithToolCallId): explanation: str = \ Field(description="An explanation to the human reviewer as to why you think" "this change is necessary and why it is safe or sound to apply it.") - + + target_path: str = Field(description=( + "The VFS path of the spec file this proposal edits. Must be one of the " + "registered spec files for this task (use list_files to see which spec " + "paths exist). The proposed contents replace this file on acceptance." + )) + state: Annotated[AIComposerState, InjectedState] @tool_display( lambda p: ( - f"Proposing spec change: {p['explanation']}" - if p.get("explanation") else "Proposing spec change" + f"Proposing spec change to {p.get('target_path', '?')}: {p['explanation']}" + if p.get("explanation") else f"Proposing spec change to {p.get('target_path', '?')}" ), None, ) @@ -73,13 +79,18 @@ class SpecChangeProposalArgs(WithToolCallId): def propose_spec_change( proposed_spec: str, explanation: str, + target_path: str, tool_call_id: Annotated[str, InjectedToolCallId], state: Annotated[AIComposerState, InjectedState] -) -> Command: +) -> Command | str: ctxt = get_runtime(AIComposerContext) - vfs_access = ctxt.context.vfs_materializer - curr_spec = vfs_access.get(state, "rules.spec") - assert curr_spec is not None + vfs_access = ctxt.context.vfs_materializer + curr_spec = vfs_access.get(state, target_path) + if curr_spec is None: + return ( + f"Target path {target_path!r} is not a registered spec file in the VFS. " + f"Use list_files to see available spec paths." + ) human_response = interrupt(ProposalType( type="proposal", proposed_spec=proposed_spec, @@ -97,7 +108,7 @@ def propose_spec_change( ) ], "vfs": { - "rules.spec": proposed_spec + target_path: proposed_spec } } ) diff --git a/composer/tools/prover.py b/composer/tools/prover.py index 0b8e57ac..afdac827 100644 --- a/composer/tools/prover.py +++ b/composer/tools/prover.py @@ -14,13 +14,18 @@ from langgraph.types import Command from composer.core.state import AIComposerState -from composer.core.context import AIComposerContext, ProverOptions, compute_state_digest -from composer.core.validation import prover as prover_key +from composer.core.context import AIComposerContext, ProverOptions, stamp +from composer.core.validation import ProverValidation from composer.prover.core import ProverReport, CexHandler, ProverOptions as CoreProverOptions, run_prover from composer.prover.callbacks import ProverEventCallbacks from composer.ui.tool_display import tool_display +# Scratch VFS path the working-spec draft is materialized to for verification +# (the draft has no committed path of its own until commit_working_spec). +_WORKING_SPEC_SCRATCH = "_composer_working.spec" + + @dataclass class ProverDeps: """Per-run dependencies injected into the prover tool — the CEX-analysis @@ -98,7 +103,19 @@ class CertoraProverTool(WithAsyncDependencies[Command, ProverDeps], WithInjected "up to date version of the code. However, when iteratively developing code, it may be useful to focus on a" "single, 'problematic' rule.") - use_working_spec : bool = Field(description="Use the working copy of the spec instead of the master copy.") + target_spec: Optional[str] = Field(default=None, description=( + "The VFS path of the committed spec file to verify against. Required " + "when use_working_spec is false; the prover reads this spec from the VFS " + "and records a per-spec completion stamp on success. Ignored when " + "use_working_spec is true (the transient working draft has no VFS path " + "until committed, and runs against it never stamp)." + )) + + use_working_spec : bool = Field(description=( + "If true, verify against the current working-spec draft instead of a " + "committed spec file; target_spec is ignored and no completion stamp is " + "recorded (the draft has no VFS path until committed)." + )) state: Annotated[AIComposerState, InjectedState] @@ -112,9 +129,12 @@ async def run(self) -> Command: # The handler already rendered the full report into result_str # (including any volume summarization it chose to do); there's no # separate truncated/summarized shape to branch on anymore. - if result.all_verified and not self.use_working_spec and not self.rule: - ctxt = get_runtime(AIComposerContext).context - state_digest = compute_state_digest(c=ctxt, state=self.state) + if ( + result.all_verified + and not self.use_working_spec + and not self.rule + and self.target_spec is not None + ): return Command( update={ "messages": [ @@ -123,9 +143,7 @@ async def run(self) -> Command: content=result.result_str ) ], - "validation": { - prover_key: state_digest - } + **stamp(ProverValidation(self.target_spec), self.state), } ) return tool_return(tool_call_id=self.tool_call_id, content=result.result_str) @@ -140,8 +158,19 @@ async def _run_certora_prover( fixed codegen ``certoraRun`` args from the tool's fields, and run. Reads everything off ``tool`` (its args map ~1:1 onto a prover run); the only runtime-context read is ``vfs_materializer``.""" - if tool.use_working_spec and not tool.state["working_spec"]: - return "No working spec written." + if tool.use_working_spec: + if not tool.state["working_spec"]: + return "No working spec written." + # The working draft has no VFS path; verify it from a scratch file + # written into the materialized tree. target_spec is ignored here. + effective_spec = _WORKING_SPEC_SCRATCH + else: + if tool.target_spec is None: + return ( + "target_spec is required when use_working_spec is false. " + "Pass the VFS path of one of the registered spec files." + ) + effective_spec = tool.target_spec ctxt = get_runtime(AIComposerContext).context writer = get_stream_writer() @@ -149,12 +178,12 @@ async def _run_certora_prover( if tool.use_working_spec: ws = tool.state["working_spec"] assert ws is not None - (Path(temp_dir) / "rules.spec").write_text(ws) + (Path(temp_dir) / _WORKING_SPEC_SCRATCH).write_text(ws) try: args = tool.source_files.copy() args.extend([ "--verify", - f"{tool.target_contract}:./rules.spec", + f"{tool.target_contract}:./{effective_spec}", "--optimistic_loop", "--optimistic_hashing", "--loop_iter", diff --git a/composer/tools/rag_env.py b/composer/tools/rag_env.py new file mode 100644 index 00000000..77395642 --- /dev/null +++ b/composer/tools/rag_env.py @@ -0,0 +1,85 @@ +"""Descriptor-driven RAG toolset selection for Rust applications. + +A wheel declares ``rag_db_default`` in its :class:`AppDescriptor`; the generic env builder looks +that tag up here and binds the corresponding corpus's search tools onto the author's env. + +Like the ecosystem registry, this maps a declarative tag → a concrete toolset; it is not an +application fork (the tool classes live in ``composer/tools/_rag.py``, shared, exactly as +``foundry_rag`` is). The tag's *connection* is not repeated here: it comes from +``composer.rag.db.KNOWLEDGE_BASES``, the same map the corpus importer +(:mod:`composer.scripts.rag_import`) targets, so a corpus is imported and searched under one name. + +**No corpus is registered yet.** Both halves of the first one — a ``composer/tools/_rag.py`` +and its ``KNOWLEDGE_BASES`` connection — land with the application that declares it, so until then +every tag is unregistered and any wheel naming one fails at descriptor load. That is the intended +resting state, not a gap: a half-registration (a tag whose tools module doesn't exist) would pass +:func:`validate_rag_db` and then be swallowed by the degrade path below, which is exactly the +confusion the two failure modes are separated to avoid. + +Two failure modes, deliberately opposite: + +* **An unregistered tag is a wheel bug.** Nothing will make that corpus appear, and degrading + silently hides the typo behind a plausible-looking run. :func:`validate_rag_db` runs at descriptor + load (:func:`composer.rustapp.host.build_application`) so it fails before the run spends anything + — the same treatment an unknown ecosystem gets. +* **An unavailable corpus is an environment condition** — the DB isn't up, the embedding model + isn't installed. A search aid must never fail a run over that, so :func:`build_rag_tools` degrades + to *no RAG* (the static cheat-sheet in the prompt suffices). +""" + +import logging +from collections.abc import Callable, Iterable +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from langchain_core.tools import BaseTool + + from composer.rag.db import ComposerRAGDB + +_log = logging.getLogger(__name__) + +#: A corpus's search-tool factory. Every import a factory needs is local to it, so the generic host +#: pulls in a corpus module only when a descriptor actually selects that corpus. +type _ToolsFactory = Callable[["ComposerRAGDB"], "Iterable[BaseTool]"] + + +#: Registered corpora, by tag. An entry is added together with the ``composer/tools/_rag.py`` +#: it imports and the ``KNOWLEDGE_BASES`` connection it needs — all three at once, or the tag +#: validates and then silently produces no tools. +_FACTORIES: dict[str, _ToolsFactory] = {} + + +def validate_rag_db(rag_db: str | None) -> None: + """Raise if ``rag_db`` names no registered corpus; ``None`` (a wheel declaring none) is fine. + + A tag needs both halves to be usable — search tools here and a connection in + ``KNOWLEDGE_BASES`` — so both are checked.""" + if rag_db is None: + return + from composer.rag.db import KNOWLEDGE_BASES + + if rag_db not in _FACTORIES or rag_db not in KNOWLEDGE_BASES: + known = sorted(_FACTORIES.keys() & KNOWLEDGE_BASES.keys()) + raise ValueError( + f"the application declares rag_db_default={rag_db!r}, which is not a registered RAG " + f"corpus ({f'known: {known}' if known else 'none is registered yet'}). Register the " + "tag in composer.rag.db.KNOWLEDGE_BASES (its connection) and composer.tools.rag_env " + "(its search tools)." + ) + + +def build_rag_tools(rag_db: str) -> "tuple[BaseTool, ...]": + """Search tools for the declared corpus, or ``()`` if it can't be opened (best-effort — the + author still has the static cheat-sheet). An unregistered tag raises; see the module docstring + for why the two are treated differently.""" + validate_rag_db(rag_db) + try: + from composer.rag.db import KNOWLEDGE_BASES, PostgreSQLRAGDatabase + from composer.rag.models import get_model + + # Lazy pool — opens on first search; the DB must already be populated. + db = PostgreSQLRAGDatabase(KNOWLEDGE_BASES[rag_db], get_model()) + return tuple(_FACTORIES[rag_db](db)) + except Exception as e: # noqa: BLE001 — RAG is optional; the cheat-sheet suffices + _log.warning("RAG %r unavailable (%s); using the static cheat-sheet only", rag_db, e) + return () diff --git a/composer/tools/result.py b/composer/tools/result.py index 0fae0538..c3c026fb 100644 --- a/composer/tools/result.py +++ b/composer/tools/result.py @@ -13,16 +13,16 @@ def check_completion( tool_call_id: str ) -> Command | None: ctxt = get_runtime(AIComposerContext).context - digest = compute_state_digest(c=ctxt, state=state) + digest = compute_state_digest(state=state) m = state.get("validation", {}) for req_v in ctxt.required_validations: - if req_v not in m or digest != m[req_v]: + if req_v.to_key() not in m or digest != m[req_v.to_key()]: return Command( update={ "messages": [ ToolMessage( tool_call_id=tool_call_id, - content=f"Result completion REJECTED; it appears you failed to satisfy the {req_v} requirement" + content=f"Result completion REJECTED; it appears you failed to satisfy the {req_v.description()} requirement" ), HumanMessage( content="You have apparently become confused about the status of your task. Evaluate the current " diff --git a/composer/tools/working_spec.py b/composer/tools/working_spec.py index 042baadc..08feb927 100644 --- a/composer/tools/working_spec.py +++ b/composer/tools/working_spec.py @@ -94,33 +94,62 @@ async def run(self) -> str | Command: ) -@tool_display("Requested spec change", None) +@tool_display( + lambda p: ( + f"Committing working spec to {p['target_path']}" + if p.get("target_path") else "Committing working spec" + ), + None, +) class CommitWorkingSpec(WithImplementation[Command | str], WithInjectedId, WithInjectedState[AIComposerState]): """ - Call this tool to ask a human reviewer to approve "committing" your working spec to the "master" copy. - - You should only use this tool after you have run the prover with sufficient rigor to confirm that the changes present - in the spec are correct and pass formal verification. In addition, the changes present here should be the minimal possible - changes to ensure the formal verification passes. Do *NOT* rewrite entire portions of the specification or make large scale changes - unless the user has explicitly approved these changes via the human_in_the_loop tool. Do *NOT* request changes that significantly - weaken the specification or otherwise trivialize it. + Call this tool to ask a human reviewer to approve "committing" your working + spec to a specific spec file in the VFS. Conceptually: ``mv + `` — the transient draft becomes the committed content at + ``target_path``. + + You should only use this tool after you have run the prover with sufficient + rigor (via ``use_working_spec=True``) to confirm that the changes present + in the spec are correct and pass formal verification. In addition, the + changes present here should be the minimal possible changes to ensure the + formal verification passes. Do *NOT* rewrite entire portions of the + specification or make large scale changes unless the user has explicitly + approved these changes via the human_in_the_loop tool. Do *NOT* request + changes that significantly weaken the specification or otherwise trivialize it. NB once the working spec has been committed to the VFS, it is discarded. + Committing does NOT by itself produce a prover verification stamp — after a + successful commit you still need to run ``certora_prover`` against the + committed spec (with ``use_working_spec=False`` and the same + ``target_spec=target_path``) to record the stamp. """ - explanation: str = \ - Field(description="An explanation to the human reviewer as to why you think the changes in the working spec" - "this change is necessary and why it is safe or sound to apply it.") + target_path: str = Field(description=( + "The VFS path of the spec file this working draft should become. Must be " + "one of the registered spec files for this task (use ``list_files`` to " + "see which spec paths exist). The working draft is written to this path " + "on acceptance." + )) + explanation: str = Field(description=( + "An explanation to the human reviewer as to why you think this change is " + "necessary and why it is safe or sound to apply it." + )) @override def run(self) -> Command | str: - if not self.state["working_spec"]: - return "No working spec set." work_spec = self.state["working_spec"] + if not work_spec: + return "No working spec set." + current = self.state.get("vfs", {}).get(self.target_path) + if current is None: + return ( + f"Target path {self.target_path!r} is not a registered spec file " + f"in the VFS. Use list_files to see available paths." + ) proposal : ProposalType = { "type": "proposal", - "current_spec": self.state["vfs"]["rules.spec"], + "current_spec": current, "proposed_spec": work_spec, - "explanation": self.explanation + "explanation": self.explanation, } response = interrupt(proposal) @@ -130,6 +159,6 @@ def run(self) -> Command | str: tool_call_id=self.tool_call_id, content="Accepted", working_spec=None, - vfs={"rules.spec": work_spec} + vfs={self.target_path: work_spec}, ) return response diff --git a/composer/ui/foundry_app.py b/composer/ui/foundry_app.py index 6d138e99..fc964a28 100644 --- a/composer/ui/foundry_app.py +++ b/composer/ui/foundry_app.py @@ -57,7 +57,7 @@ class FoundryTaskHandler(MultiJobTaskHandler[None], NullEventHandler): """Per-task handler that doubles as its own ``EventHandler``. Streams ``forge_test_run`` summaries into a collapsible ``RichLog`` - mounted under the task panel. + pinned at the top of the task panel. """ def __init__( @@ -81,9 +81,7 @@ async def _ensure_forge_log(self) -> RichLog: log = RichLog(highlight=True, markup=False) log.styles.min_height = 15 self._forge_log = log - await self._mount_to( - self._panel, Collapsible(log, title="Forge Test Runs"), - ) + await self._mount_fixture(Collapsible(log, title="Forge Test Runs")) return self._forge_log @override diff --git a/composer/ui/message_renderer.py b/composer/ui/message_renderer.py index bbb04653..df25d2b0 100644 --- a/composer/ui/message_renderer.py +++ b/composer/ui/message_renderer.py @@ -24,6 +24,7 @@ from graphcore.graph import INITIAL_NODE, TOOL_RESULT_NODE, TOOLS_NODE from graphcore.utils import NormalizedTokenUsage, get_normalized_token_usage +from composer.llm.pricing import price_per_mtok KNOWN_NODES: set[str] = {INITIAL_NODE, TOOL_RESULT_NODE, TOOLS_NODE} @@ -42,121 +43,6 @@ def dot(style: str, text: Text | str) -> Text: return result -@dataclass(frozen=True) -class _PriceTier: - """Per-million-token prices in USD for one model at one - context tier. - - ``input`` is the price for *fresh* input tokens (the bucket left - after subtracting ``cache_read`` and ``cache_write`` from the - total). ``output`` is the price for output tokens, which on the - OpenAI side already includes reasoning tokens (billed at the - output rate by both providers). ``cache_read`` / ``cache_write`` - are the cache-bucket rates. - - Anthropic-specific note: the ``cache_write`` rate here is the - 5-minute ephemeral rate. Anthropic also has a 1-hour ephemeral - rate that's higher (~2× base input); our normalized usage rolls - both into a single ``cache_write_tokens`` bucket, so workloads - that heavily use 1h caching will be slightly under-billed by - this banner. Approximation is fine for a banner.""" - input: float - output: float - cache_read: float - cache_write: float - - -@dataclass(frozen=True) -class _ModelPricing: - """Pricing entry for one model family. ``long`` is the - long-context tier (used when input token count exceeds the - threshold) and applies only to OpenAI models that publish a - separate >272K-input rate; Anthropic models keep ``long = None`` - and bill everything at ``short`` rates.""" - short: _PriceTier - long: _PriceTier | None = None - - -# OpenAI's published >272K input-token threshold for long-context -# pricing. Once an individual call's input crosses this, the long -# tier applies *for the full session* per OpenAI's terms; we -# approximate that by switching on a per-message basis (a session -# that drifts above 272K will mostly stay there). -_OPENAI_LONG_CONTEXT_THRESHOLD = 272_000 - - -# Pricing tables transcribed from Anthropic + OpenAI rate cards. -# Sources should be re-checked when new model families ship. -_PRICING: list[tuple[str, _ModelPricing]] = [ - # ---- Anthropic ---- - # Fable is its own rate card, a tier above Opus. - ("claude-fable-5", _ModelPricing(short=_PriceTier(10.00, 50.00, 1.00, 12.50))), - - # claude-opus-5 and claude-opus-4.5 / 4.6 / 4.7 / 4.8 share a rate card; - # older 4 / 4.1 are pricier. Matching by prefix-of-prefix so - # "claude-opus-4-7" and "claude-opus-4-7-20260301" both hit the right entry. - ("claude-opus-5", _ModelPricing(short=_PriceTier(5.00, 25.00, 0.50, 6.25))), - ("claude-opus-4-8", _ModelPricing(short=_PriceTier(5.00, 25.00, 0.50, 6.25))), - ("claude-opus-4-7", _ModelPricing(short=_PriceTier(5.00, 25.00, 0.50, 6.25))), - ("claude-opus-4-6", _ModelPricing(short=_PriceTier(5.00, 25.00, 0.50, 6.25))), - ("claude-opus-4-5", _ModelPricing(short=_PriceTier(5.00, 25.00, 0.50, 6.25))), - ("claude-opus-4-1", _ModelPricing(short=_PriceTier(15.00, 75.00, 1.50, 18.75))), - ("claude-opus-4", _ModelPricing(short=_PriceTier(15.00, 75.00, 1.50, 18.75))), - - ("claude-sonnet-5", _ModelPricing(short=_PriceTier(3.00, 15.00, 0.30, 3.75))), - ("claude-sonnet-4-6", _ModelPricing(short=_PriceTier(3.00, 15.00, 0.30, 3.75))), - ("claude-sonnet-4-5", _ModelPricing(short=_PriceTier(3.00, 15.00, 0.30, 3.75))), - ("claude-sonnet-4", _ModelPricing(short=_PriceTier(3.00, 15.00, 0.30, 3.75))), - - ("claude-haiku-4-5", _ModelPricing(short=_PriceTier(1.00, 5.00, 0.10, 1.25))), - - # ---- OpenAI ---- - # gpt-5.5 / 5.4 publish short (≤272K input) and long (>272K) tiers. - # Pro variants don't publish a cached-in discount (cache_read = - # base input). Mini/nano don't publish a long tier; we use short - # for everything on those. - ("gpt-5.5-pro", _ModelPricing( - short=_PriceTier(30.00, 180.00, 30.00, 30.00), - long=_PriceTier(60.00, 270.00, 60.00, 60.00), - )), - ("gpt-5.5", _ModelPricing( - short=_PriceTier(5.00, 30.00, 0.50, 5.00), - long=_PriceTier(10.00, 45.00, 1.00, 10.00), - )), - ("gpt-5.4-pro", _ModelPricing( - short=_PriceTier(30.00, 180.00, 30.00, 30.00), - long=_PriceTier(60.00, 270.00, 60.00, 60.00), - )), - ("gpt-5.4-mini", _ModelPricing(short=_PriceTier(0.75, 4.50, 0.075, 0.75))), - ("gpt-5.4-nano", _ModelPricing(short=_PriceTier(0.20, 1.25, 0.02, 0.20))), - ("gpt-5.4", _ModelPricing( - short=_PriceTier(2.50, 15.00, 0.25, 2.50), - long=_PriceTier(5.00, 22.50, 0.50, 5.00), - )), -] - - -def _price_per_mtok(model: str | None, input_tokens: int) -> _PriceTier | None: - """Look up per-MTok pricing by model name and call size. Returns - ``None`` for models with no table entry (cost contribution becomes - zero — better than guessing). - - Matched by prefix on the lowercased model name so dated revisions - (``claude-opus-4-7-20260301``, ``gpt-5.5-2026-...``) collapse into - the same family entry. Table is searched in order, so list more - specific prefixes (``gpt-5.5-pro``) before less specific - (``gpt-5.5``). For OpenAI models with a long tier, ``input_tokens`` - chooses short vs. long; Anthropic always uses the short tier.""" - if model is None: - return None - m = model.lower() - for prefix, pricing in _PRICING: - if m.startswith(prefix): - if pricing.long is not None and input_tokens > _OPENAI_LONG_CONTEXT_THRESHOLD: - return pricing.long - return pricing.short - return None - class TokenStats: """Accumulates token usage across AI messages and updates a display widget. @@ -183,7 +69,7 @@ def __init__(self, display: Static): @staticmethod def _cost_of(usage: NormalizedTokenUsage) -> float: - price = _price_per_mtok(usage["model_name"], usage["total_input_tokens"]) + price = price_per_mtok(usage["model_name"], usage["total_input_tokens"]) if price is None: return 0.0 # Fresh (non-cached) input is the total minus the cached diff --git a/composer/ui/multi_job_app.py b/composer/ui/multi_job_app.py index 54563574..d2065632 100644 --- a/composer/ui/multi_job_app.py +++ b/composer/ui/multi_job_app.py @@ -409,6 +409,14 @@ async def _mount_to(self, target: VerticalScroll, *widgets: Widget) -> None: if target.max_scroll_y - target.scroll_y <= 3: target.scroll_end(animate=False) + async def _mount_fixture(self, widget: Widget) -> None: + """Mount a task-lifetime fixture: pinned above the conversation stream as the + panel's first child, rather than appended at whatever stream position it was + first needed. For widgets that accumulate for the task's whole life (e.g. a + streaming event log); output anchored to one moment in the conversation + belongs in ``_mount_to``.""" + await self._panel.mount(widget, before=0) + # ── Content links ─────────────────────────────────────── async def render_content_link(self, label: str, content: str, filename: str) -> None: @@ -667,6 +675,14 @@ async def _mount_content_pane(self, label: str, content: str, filename: str) -> self._previous_view = switcher.current switcher.current = pane_id + def mark_pipeline_done(self) -> None: + """The run has ended — successfully or not — so quitting is allowed from here on. + + Every entry point's worker calls this in *both* its success and its failure path: until it + does, :meth:`action_quit_app` swallows the quit key, so the user can't close the app (and + lose every panel with it) while work is still streaming in.""" + self._pipeline_done = True + def action_quit_app(self) -> None: if self._pipeline_done: self.exit() diff --git a/composer/ui/pipeline_app.py b/composer/ui/pipeline_app.py index ba7e9353..1240476b 100644 --- a/composer/ui/pipeline_app.py +++ b/composer/ui/pipeline_app.py @@ -143,7 +143,7 @@ async def on_pipeline_done(self, result: PipelineResult) -> None: stub, and specs under ``/natspec_output/`` (defaulting to ``cwd/natspec_output/`` when no ``--output-root`` was passed). """ - self._pipeline_done = True + self.mark_pipeline_done() summary = self.query_one("#summary", VerticalScroll) switcher = self.query_one("#switcher", ContentSwitcher) @@ -173,7 +173,7 @@ async def mount_error(self, exc: Exception) -> None: ``on_pipeline_done`` uses) and renders the exception + traceback, so per-task error panels stay visible alongside the top-level cause. """ - self._pipeline_done = True + self.mark_pipeline_done() summary = self.query_one("#summary", VerticalScroll) switcher = self.query_one("#switcher", ContentSwitcher) switcher.current = "summary" diff --git a/composer/ui/tool_display.py b/composer/ui/tool_display.py index 92c706c7..64c55913 100644 --- a/composer/ui/tool_display.py +++ b/composer/ui/tool_display.py @@ -1,11 +1,14 @@ from langchain_core.messages import ToolMessage from dataclasses import dataclass, field -from typing import Callable +from typing import Any, Callable, Concatenate from contextvars import ContextVar from contextlib import contextmanager, asynccontextmanager +type ToolDisplayFamily[**P] = Callable[Concatenate[dict, P], str] | str +type ResultFamilyDisplay[**P] = str | Callable[Concatenate[str, ToolMessage, P], str | None | tuple[str, str]] | None + type DisplayLabelTy = Callable[[dict], str] | str type ResultOutputTy = str | Callable[[str, ToolMessage], str | None | tuple[str, str]] | None @@ -365,7 +368,7 @@ def _register_tool_spec( from typing import TypeVar -from graphcore.tools.schemas import WithAsyncDependencies, WithAsyncImplementation, WithImplementation, ToolBuilder +from graphcore.tools.schemas import ToolFamilyParams, WithAsyncDependencies, WithAsyncImplementation, WithImplementation, ToolBuilder, _TemplatedTool, tool_family from langchain_core.tools import BaseTool from functools import wraps @@ -430,3 +433,42 @@ def tool_display( short_display_name=short_label, ) ) + +def _inject_params_call[**P]( + m: ToolDisplayFamily[P], + *args: P.args, + **kwargs: P.kwargs +) -> DisplayLabelTy: + if isinstance(m, str): + return m + else: + return lambda tc_dict: m(tc_dict, *args, **kwargs) + +def _inject_params_result[**P]( + m: ResultFamilyDisplay[P], + *args: P.args, + **kwargs: P.kwargs +) -> ResultOutputTy: + if isinstance(m, str) or m is None: + return m + return lambda nm, res_msg: m(nm, res_msg, *args, **kwargs) + +def tool_family_display[**P, T: type[WithAsyncDependencies] | type[WithAsyncImplementation] | type[WithImplementation]]( + label: ToolDisplayFamily[P], + result: ResultFamilyDisplay[P], + short_label: ToolDisplayFamily[P] | None = None +) -> Callable[[type[_TemplatedTool[T, Any, P]]], type[_TemplatedTool[T, Any, P]]]: + def wrapper( + f: type[_TemplatedTool[T, Any, P]] + ) -> type[_TemplatedTool[T, Any, P]]: + old_with_template = f.with_template + @wraps(old_with_template) + def new_with_template(*args: P.args, **kwargs: P.kwargs) -> T: + return tool_display( + _inject_params_call(label, *args, **kwargs), + _inject_params_result(result, *args, **kwargs), + _inject_params_call(short_label, *args, **kwargs) if short_label is not None else None, + )(old_with_template(*args, **kwargs)) + f.with_template = new_with_template + return f + return wrapper diff --git a/composer/workflow/executor.py b/composer/workflow/executor.py index 6281993c..5e1a920b 100644 --- a/composer/workflow/executor.py +++ b/composer/workflow/executor.py @@ -4,7 +4,6 @@ from dataclasses import dataclass import pathlib -from langchain_core.runnables import RunnableConfig from langchain_core.tools import BaseTool from graphcore.graph import Builder @@ -30,7 +29,7 @@ from composer.core.context import AIComposerContext, ProverOptions from composer.core.state import AIComposerState from composer.prover.core import make_prover_options, CexHandler -from composer.core.validation import ValidationType, prover, reqs as req_type +from composer.core.validation import ProverValidation, CodegenValidation, ReqsValidation from composer.rag.db import rag_context, ComposerRAGDB from composer.rag.models import get_model as get_rag_model from composer.audit.store import AuditStore, ResumeArtifact @@ -41,7 +40,7 @@ from composer.tools.search import cvl_manual_tools from composer.templates.loader import load_jinja_template from composer.io.protocol import CodeGenIOHandler, WorkflowPurpose -from composer.io.context import with_handler, run_graph +from composer.io.context import with_handler, run_to_completion from composer.diagnostics.timing import set_current_task_id from composer.ui.codegen_events import CodeGenEventHandler from composer.core.state import AIComposerInput, AIComposerExtra @@ -78,7 +77,7 @@ class _ExecutorOptions(WorkflowOptions, ModelOptionsBase, Protocol): def get_reference_input(input_data: InputData, debug_prompt: Optional[str]) -> str: return load_jinja_template( "workflow_info.j2", - spec_filename=input_data.spec.basename, + spec_filenames=[s.file.basename for s in input_data.specs], interface_filename=input_data.intf.basename, system_doc_filename=input_data.system_doc.basename, debug_prompt=debug_prompt) @@ -90,15 +89,17 @@ def _get_empty_extra() -> AIComposerExtra: def get_fresh_input(input: InputData, workflow_options: WorkflowOptions) -> AIComposerInput: - return AIComposerInput(input=[ - input.intf.to_dict(), - input.spec.to_dict(), - input.system_doc.to_dict(), - { - "type": "text", - "text": get_reference_input(input_data=input, debug_prompt=workflow_options.debug_prompt_override) - } - ], vfs={"rules.spec": input.spec.string_contents}, **_get_empty_extra()) + messages: list[str | dict] = [input.intf.to_dict()] + vfs: dict[str, str] = {} + for s in input.specs: + messages.append(s.file.to_dict()) + vfs[s.vfs_path] = s.file.string_contents + messages.append(input.system_doc.to_dict()) + messages.append({ + "type": "text", + "text": get_reference_input(input_data=input, debug_prompt=workflow_options.debug_prompt_override), + }) + return AIComposerInput(input=messages, vfs=vfs, **_get_empty_extra()) @dataclass class InputChangeDesc: @@ -110,10 +111,36 @@ class InputChangeDesc: vfs_note: Optional[str] + +@dataclass +class SpecDelta: + """A registered spec's before/after pair for resume-prompt rendering.""" + vfs_path: str + orig_text: str + updated_text: str + + +def _resume_spec_srcs( + resume_art: ResumeArtifact, new_specs: dict[str, TextNativeFS] +) -> list[tuple[str, TextUploadable]]: + """Resume-time source for each registered spec: the updated disk file if one + was supplied for that path, else the prior committed contents.""" + out: list[tuple[str, TextUploadable]] = [] + for path in resume_art.spec_vfs_paths: + updated = new_specs.get(path) + if updated is not None: + out.append((path, updated)) + else: + prior = resume_art.spec_at(path) + assert prior is not None + out.append((path, prior)) + return out + + def get_resume_prompt_common( art: ResumeArtifact, res: ResumeInput, - updated_spec: str, + spec_deltas: list[SpecDelta], other_changes: list[InputChangeDesc] | None = None ) -> list[str | dict]: changes = [] @@ -135,37 +162,60 @@ def get_resume_prompt_common( "resume_prompt.j2", commentary=art.commentary, spec_change_commentary=res.comments, - orig_spec=art.spec.contents, - new_spec=updated_spec, + spec_deltas=[ + {"vfs_path": d.vfs_path, "orig": d.orig_text, "new": d.updated_text} + for d in spec_deltas + ], other_changes=changes )] def get_resume_id_input(input: ResumeIdData, resume_art: ResumeArtifact, workflow_options: WorkflowOptions) -> AIComposerInput: + vfs_materialize = resume_art.vfs.to_dict() + new_vfs = { k: v.decode("utf-8") for (k, v) in vfs_materialize.items() } + + spec_deltas: list[SpecDelta] = [] + for vfs_path, new_file in input.new_specs.items(): + new_text = new_file.string_contents + new_vfs[vfs_path] = new_text + prior = resume_art.spec_at(vfs_path) + spec_deltas.append(SpecDelta( + vfs_path=vfs_path, + orig_text=prior.contents if prior is not None else f"[new spec {vfs_path}]", + updated_text=new_text, + )) - input_messages : list[str | dict] = get_resume_prompt_common( + input_messages = get_resume_prompt_common( art=resume_art, res=input, - updated_spec=input.new_spec.string_contents + spec_deltas=spec_deltas, ) if workflow_options.debug_prompt_override is not None: input_messages.append(workflow_options.debug_prompt_override) - vfs_materialize = resume_art.vfs.to_dict() - new_vfs = { k: v.decode("utf-8") for (k, v) in vfs_materialize.items() } - new_vfs["rules.spec"] = input.new_spec.string_contents return AIComposerInput( input=input_messages, vfs=new_vfs, **_get_empty_extra() ) -def get_resume_fs_input(input: ResumeFSData, resume_art: ResumeArtifact, workflow_options: WorkflowOptions) -> tuple[AIComposerInput, TextNativeFS, TextNativeFS]: +def get_resume_fs_input( + input: ResumeFSData, + resume_art: ResumeArtifact, + workflow_options: WorkflowOptions +) -> tuple[AIComposerInput, TextNativeFS, list[tuple[str, TextUploadable]]]: path = pathlib.Path(input.file_path) - spec_p = path / "rules.spec" - if not spec_p.is_file(): - raise RuntimeError("Specification file is apparently missing") - new_spec = spec_p.read_text() + spec_srcs: list[tuple[str, TextUploadable]] = [] + spec_deltas: list[SpecDelta] = [] + for vfs_path in resume_art.spec_vfs_paths: + spec_p = path / vfs_path + if not spec_p.is_file(): + raise RuntimeError(f"Spec file missing on resume: expected {spec_p} to exist") + spec_srcs.append((vfs_path, TextNativeFS(spec_p))) + new_text = spec_p.read_text() + prior = resume_art.spec_at(vfs_path) + if prior is not None and new_text != prior.contents: + spec_deltas.append(SpecDelta(vfs_path=vfs_path, orig_text=prior.contents, updated_text=new_text)) intf_p = path / resume_art.interface_path if not intf_p.is_file(): @@ -182,8 +232,8 @@ def get_resume_fs_input(input: ResumeFSData, resume_art: ResumeArtifact, workflo input_messages = get_resume_prompt_common( art=resume_art, res=input, + spec_deltas=spec_deltas, other_changes=changes, - updated_spec=new_spec ) input_messages.append("In addition to the explicit changes mentioned above, the contents of the VFS may have been arbitrarily changed since your last work. " \ "Some of these changes may cause the current implementation to no longer compile. Thus, analyze the current implementation and consider what changes are necessary to " \ @@ -192,7 +242,7 @@ def get_resume_fs_input(input: ResumeFSData, resume_art: ResumeArtifact, workflo if workflow_options.debug_prompt_override is not None: input_messages.append(workflow_options.debug_prompt_override) - return (AIComposerInput(input=input_messages, vfs={}, **_get_empty_extra()), TextNativeFS(intf_p), TextNativeFS(spec_p)) + return (AIComposerInput(input=input_messages, vfs={}, **_get_empty_extra()), TextNativeFS(intf_p), spec_srcs) async def execute_ai_composer_workflow( @@ -261,8 +311,12 @@ async def _run_codegen( # provider ``Document``s via the connection's FileUploader. system_doc_doc: Document interface_doc: TextDocument - spec_doc: TextDocument + # (vfs_path, document) per registered spec. On a fresh run these are the + # uploaded inputs directly; on resume the audit / disk handles are + # ``Uploadable`` and get rehydrated into ``Document``s via the uploader. + spec_docs: list[tuple[str, TextDocument]] resume_art : None | ResumeArtifact = None + match input: case InputData(): @@ -270,7 +324,7 @@ async def _run_codegen( flow_input = get_fresh_input(input, workflow_options) system_doc_doc = input.system_doc interface_doc = input.intf - spec_doc = input.spec + spec_docs = [(s.vfs_path, s.file) for s in input.specs] case ResumeIdData() | ResumeFSData(): prompt_params = PromptParams(is_resume=True) @@ -281,17 +335,21 @@ async def _run_codegen( ) system_doc_doc = await conn.uploader.document_from(system_src) intf_src: TextUploadable - spec_src: TextUploadable + spec_srcs: list[tuple[str, TextUploadable]] match input: case ResumeFSData(): - (flow_input, intf_src, spec_src) = get_resume_fs_input(input, resume_art, workflow_options) + (flow_input, intf_src, spec_srcs) = get_resume_fs_input(input, resume_art, workflow_options) fs_layer = input.file_path case ResumeIdData(): intf_src = resume_art.intf_vfs_handle - spec_src = input.new_spec + spec_srcs = _resume_spec_srcs(resume_art, input.new_specs) flow_input = get_resume_id_input(input, resume_art, workflow_options) interface_doc = conn.uploader.text_document_from(intf_src) - spec_doc = conn.uploader.text_document_from(spec_src) + spec_docs = [(p, conn.uploader.text_document_from(s)) for (p, s) in spec_srcs] + + # One prover completion gate per registered spec (the reqs gate is appended + # below iff requirements were extracted). + # flow_input["required_validations"] = [ProverValidation(p) for (p, _) in spec_docs] req_mem_tool = conn.memory(get_memory_ns(mem_root, "natreq")) @@ -303,7 +361,7 @@ async def _run_codegen( workflow_options, llm.builder_for(), system_doc_doc, - spec_doc, + [d for (_, d) in spec_docs], req_mem_tool, resume_art, ) @@ -372,8 +430,7 @@ async def _run_codegen( await audit_store.register_run( thread_id=thread_id, - spec_vfs_path="rules.spec", - spec_file=spec_doc, + specs=spec_docs, interface_file=interface_doc, system_doc=system_doc_doc, vfs_init=materializer.iterate(flow_input), @@ -408,15 +465,9 @@ async def _run_codegen( except ModuleNotFoundError: pass - config: RunnableConfig = {"configurable": {"thread_id": thread_id}} - config["recursion_limit"] = workflow_options.recursion_limit - - if workflow_options.checkpoint_id is not None: - config["configurable"]["checkpoint_id"] = workflow_options.checkpoint_id - - required_validations: list[ValidationType] = [prover] + required_validations: list[CodegenValidation] = [ProverValidation(s) for (s, _) in spec_docs] if reqs_list is not None: - required_validations.append(req_type) + required_validations.append(ReqsValidation()) work_context = AIComposerContext( vfs_materializer=materializer, required_validations=required_validations @@ -425,7 +476,15 @@ async def _run_codegen( try: async with with_handler(handler, CodeGenEventHandler(handler)): with set_current_task_id(CODEGEN_TASK_ID): - final_state = await run_graph(workflow_exec, work_context, flow_input, config, description="Code generation") + final_state = await run_to_completion( + workflow_exec, + flow_input, + thread_id=thread_id, + context=work_context, + checkpoint_id=workflow_options.checkpoint_id, + recursion_limit=workflow_options.recursion_limit, + description="Code generation", + ) result = final_state.get("generated_code", None) if result is None: diff --git a/composer/workflow/factories.py b/composer/workflow/factories.py index aa0b9c8b..9c3ba9f1 100644 --- a/composer/workflow/factories.py +++ b/composer/workflow/factories.py @@ -22,14 +22,17 @@ def get_vfs_tools( return vfs_tools(VFSToolConfig( fs_layer=fs_layer, immutable=False, - forbidden_write="^rules.spec$", + # Block writes to ANY spec file. Spec mutations must go through + # propose_spec_change (committed edits) or write_working_spec + + # commit_working_spec (iterative drafts). + forbidden_write=r"^.+\.spec$", put_doc_extra= \ """ By convention, every Solidity file placed into the virtual filesystem should contain exactly one contract/interface/library definitions. Further, the name of the contract/interface/library defined in that file should name the name of the solidity source file sans extension. For example, src/MyContract.sol should contain an interface/library/contract called `MyContract`" - IMPORTANT: You may not use this tool to update the specification, nor should you attempt to - add new specification files. + IMPORTANT: You may not use this tool to update, create, or delete any spec file (any path ending in `.spec`). + All spec mutations must go through propose_spec_change or the write_working_spec / commit_working_spec flow. """ ), AIComposerState) diff --git a/docs/application-abstraction.md b/docs/application-abstraction.md new file mode 100644 index 00000000..816d108c --- /dev/null +++ b/docs/application-abstraction.md @@ -0,0 +1,551 @@ +# Design Doc — What Is an AutoProver "Application" + +> How the pieces — argument parsing, service setup, the pipeline, and the UI — are +> wired into a single, runnable *application* such as **autoprove** or **foundry**, +> and the conventions a new application is expected to follow. +> +> Companion to [ARCHITECTURE.md](../ARCHITECTURE.md) and +> [formalization-abstraction.md](./formalization-abstraction.md). Where the +> formalization doc zooms into the *backend* seam (how a property becomes a verified +> artifact), this doc zooms out to the *whole vertical*: everything from `argv` to a +> rendered TUI. The [MultiJobApp design](../composer/ui/MULTI_JOB_DESIGN.md) covers +> the generic UI base this leans on. + +--- + +## 1. What "application" means here + +An AutoProver **application** is a complete, runnable vertical slice that takes a +Solidity project + a design document and drives the shared property-extraction / +formalization pipeline to a set of on-disk deliverables, rendered live to the user. + +Two ship as hand-written verticals: + +| Application | Deliverable | Backend | Entry points | +|---|---|---|---| +| **autoprove** | CVL `.spec` + `.conf`, verified by the Certora Prover | `ProverBackend` | [tui_autoprove.py](../composer/cli/tui_autoprove.py) · [console_autoprove.py](../composer/cli/console_autoprove.py) | +| **foundry** | `.t.sol` tests, gated by `forge test` | `FoundryBackend` | [tui_foundry.py](../composer/cli/tui_foundry.py) · [console_foundry.py](../composer/cli/console_foundry.py) | + +Crucially, "application" is **not** a single class. It is a *convention*: a set of +five collaborating pieces, each an implementation of a shared abstraction, wired +together by a thin `main()`. The value of the convention is that the pieces are +mutually orthogonal — you can swap the frontend (TUI ↔ console) without touching the +pipeline, and swap the backend without touching either frontend. + +There is also a third path that writes **none** of the five by hand: an application whose backend +is a Rust wheel *declares* them in an `AppDescriptor`, and the generic host in +[composer/rustapp/](../composer/rustapp/) synthesizes the phase enum, entry point, frontend, store +and `main()` from that declaration. The convention below is what that host implements, so this doc +is still the model — see [rust-applications.md](./rust-applications.md) for the declarative form. + +--- + +## 2. The five pieces of an application + +Every application is assembled from exactly these, each keyed off one shared +type parameter — the application's **phase enum** `P`: + +``` + ┌─────────────────────────────────────────────────────────────┐ + │ main() (composer/cli/*.py) │ + │ async with entry_point(summary) as run: ← the Executor │ + │ app = FrontendApp() ← the Frontend │ + │ await run(app.make_handler) ← the seam │ + └─────────────────────────────────────────────────────────────┘ + │ │ + ┌───────────▼────────────┐ ┌─────────────▼─────────────┐ + │ 2. Entry point / │ │ 4. Frontend │ + │ Executor │ │ MultiJobApp[P, H] │ + │ argv → services → │ │ OR console handler │ + │ a run(handler) closure│ │ supplies make_handler: │ + └───────────┬────────────┘ │ HandlerFactory[P, H] │ + │ calls └───────────────────────────┘ + ┌───────────▼───────────┐ + │ 3. Pipeline │ 1. Phase enum P: HasName + │ run_pipeline(backend)│ (the spine that threads all + │ + a PipelineBackend │ five together) + └───────────┬───────────┘ + │ contributes + ┌───────────▼───────────┐ + │ 5. Artifact store │ + │ on-disk layout │ + └───────────────────────┘ +``` + +1. **Phase enum** `P` — the task-grouping vocabulary. +2. **Entry point / Executor** — argv → configured services → a `run(handler)` closure. +3. **Pipeline + backend** — the work, expressed as a `PipelineBackend` fed to the + shared `run_pipeline` driver. +4. **Frontend** — a `MultiJobApp[P, H]` subclass (TUI) or console handler that + supplies a `HandlerFactory[P, H]`. +5. **Artifact store** — the on-disk deliverable layout. + +The rest of this doc walks each piece with the autoprove and foundry +implementations side by side. + +--- + +## 3. The seam that makes it compose: `HandlerFactory` + +Before the pieces, understand the seam between them. The pipeline and the frontend +never reference each other. They meet at one protocol, +[`HandlerFactory[P, H]`](../composer/io/multi_job.py): + +```python +# composer/io/multi_job.py +class HandlerFactory[P: HasName, H](Protocol): + def __call__(self, /, info: TaskInfo[P]) -> Awaitable[TaskHandle[H]]: ... +``` + +- The **pipeline** is a producer of *work*. Every task it launches is described by a + `TaskInfo(task_id, label, phase)` and run through `run_task`, which calls the factory + to obtain a `TaskHandle` (an `IOHandler` + `EventHandler` + lifecycle callbacks). +- The **frontend** is a producer of *handlers*. It implements the factory: given a + `TaskInfo`, it mounts a panel, builds a per-task renderer, and returns the + `TaskHandle`. + +So the entire application boils down to: + +```python +async with entry_point(summary) as run: # run: Executor (the pipeline, service-loaded) + app = FrontendApp() # the frontend + result = await run(app.make_handler) # hand the factory to the pipeline +``` + +`run` is typed as an **Executor**, and its whole signature *is* the seam: + +```python +# composer/spec/source/autoprove_common.py +type Executor = Callable[[HandlerFactory[AutoProvePhase, None]], Awaitable[CorePipelineResult[GeneratedCVL]]] +# composer/foundry/pipeline.py +type FoundryPipelineExecutor = Callable[[HandlerFactory[FoundryPhase, None]], Awaitable[FoundryPipelineResult]] +``` + +Because the pipeline only ever *calls* the factory and the frontend only ever +*implements* it, the two are swappable independently. That is why autoprove has both +a TUI ([`AutoProveApp`](../composer/ui/autoprove_app.py)) and a console +([`AutoProveConsoleHandler`](../composer/ui/autoprove_console.py)) frontend against +the same pipeline, selected purely by which `make_handler` `main()` passes in. + +`H` is the human-interaction schema. Both current applications are non-interactive at +the per-task level (`H = None`; their handlers raise from `format_hitl_prompt`), but +the seam carries the type so an interactive application (e.g. the NatSpec pipeline) +plugs in without changing the contract. + +--- + +## 4. Piece 1 — the phase enum `P` + +Every application defines a single enum whose members are its task-grouping phases. +This enum is the type parameter that threads through the frontend +(`MultiJobApp[P, ...]`), the seam (`HandlerFactory[P, H]`, `TaskInfo[P]`), and the +backend (`CorePhases[P]`). It only needs to satisfy `HasName` (an enum trivially does). + +```python +# composer/ui/autoprove_app.py +class AutoProvePhase(enum.Enum): + DISCOVER_DESIGN_DOC = "discover_design_doc" + HARNESS = "harness" + AUTOSETUP = "autosetup" + INVARIANTS = "invariants" + SUMMARIES = "summaries" + COMPONENT_ANALYSIS = "component_analysis" + BUG_ANALYSIS = "bug_analysis" + CVL_GEN = "cvl_gen" + REPORT = "report" +``` + +```python +# composer/foundry/pipeline.py +class FoundryPhase(enum.Enum): + DISCOVER_DESIGN_DOC = "discover_design_doc" + SYSTEM_ANALYSIS = "system_analysis" + PROPERTY_EXTRACTION = "property_extraction" + TEST_GENERATION = "test_generation" + REPORT = "report" +``` + +Both carry `DISCOVER_DESIGN_DOC` because design-doc discovery is a *pre-pipeline* task the shared +entry point runs (only when `system_doc` was omitted), and it still needs a phase to be grouped +under — the application passes the member as `cli_pipeline`'s `design_doc_phase` (§5). + +The phase serves two roles: + +- **Grouping in the UI.** The frontend maps each phase to a human label and an + ordering, and every task lands in the section for its phase: + + ```python + # composer/ui/foundry_app.py + FOUNDRY_PHASE_LABELS = { + FoundryPhase.DISCOVER_DESIGN_DOC: "Design Doc Discovery", + FoundryPhase.SYSTEM_ANALYSIS: "System Analysis", + FoundryPhase.PROPERTY_EXTRACTION: "Property Extraction", + FoundryPhase.TEST_GENERATION: "Test Generation", + } + FOUNDRY_SECTION_ORDER = [ + "Design Doc Discovery", "System Analysis", "Property Extraction", "Test Generation", + ] + ``` + + A phase may be absent from the labels (foundry's `REPORT` is): the label map drives *sections*, + so an unlabelled phase simply gets no section of its own. + +- **The driver ↔ backend contract.** The shared driver tags four *core* phases; the + backend maps its own enum onto them via `CorePhases[P]` (see §6). Note the two enums + above differ in granularity: foundry has five phases, autoprove has nine — the + prover contributes several extra prep phases (harness, autosetup, summaries, + invariants) that the driver never knows about. The enum is the application's own + vocabulary; only the four core slots are shared. + +--- + +## 5. Piece 2 — the entry point / Executor + +Each application has an `_entry_point` async context manager that yields the Executor closure. It +owns only what is *its own* — the argument parser and the choice of env/backend/ecosystem — and +delegates all the imperative service setup to one shared context manager, +[`cli_pipeline`](../composer/pipeline/cli.py): + +> parse args → `cli_pipeline` (services, design-doc resolution, cache root) → build this +> application's env + backend → hand them to the continuation. + +```python +# composer/foundry/entry.py (shape shared by composer/spec/source/autoprove_common.py) +@asynccontextmanager +async def _entry_point(summary: RunSummary) -> AsyncIterator[FoundryRunner]: + args = cast(FoundryArgs, _build_parser().parse_args()) + thread_id = f"foundry_{uuid.uuid4().hex[:12]}" + + async def runner(fact: HandlerFactory[FoundryPhase, None]) -> FoundryPipelineResult: + async with ( + cli_pipeline( + args=args, thread_id=thread_id, summary=summary, task_handler=fact, + design_doc_phase=FoundryPhase.DISCOVER_DESIGN_DOC, + at_exit=_usage_exit_logger(summary), workflow="foundry", + ) as (staged, cont), + PostgreSQLRAGDatabase.rag_context(staged.embed_model, args.rag_db) as rag, + ): + env = build_foundry_env(model_provider=staged.llm_models, rag_db=rag, ...) + return await cont(env, backend(source_input=staged.source, ...), EVM) + + yield runner +``` + +`cli_pipeline` is the shared half — one implementation, not a per-application convention: + +1. Resolves `project_root` + `main_contract` (`path:ContractName`). +2. Opens the shared connection stack — `standard_connections`, the async tool context, the + thread logger — under a single `async with`. +3. **Resolves the design doc**: the supplied `system_doc`, or, when it was omitted, discovers one + as a visible task tagged with the caller's `design_doc_phase`. This is why discovery needs a + phase member (§4) and why it happens inside the handler scope. +4. Computes the **root cache key** (project root + doc bytes + relative path + contract name), + records the run's cache tags, and reads an optional threat model. +5. Yields `(staged, cont)`: a `StagedPipeline` of everything the application needs to build its + env and backend (connections, models, embedder, `SourceCode`, logger, `root_key`), and a + `Continuation` that takes `(env, backend, ecosystem)`, builds the `PipelineRun`, and runs the + driver. + +What stays per-application is exactly the part that differs: the parser, the tool/RAG env, the +backend, and which ecosystem it targets. + +The args are declared as a **Protocol** (`AutoProveArgs`, `FoundryArgs`), not a class, +so the parser and the typed access agree without a dataclass in between: + +The shared half declares what it needs of them as `PipelineArgs` +([pipeline/cli.py](../composer/pipeline/cli.py)) — `project_root`, `main_contract`, `system_doc`, +the cache/memory namespaces, concurrency, recursion limit, `interactive`, `threat_model`, +`max_bug_rounds` — and each application's protocol extends that with its own: + +```python +# composer/spec/source/autoprove_common.py +class AutoProveArgs(ExtendedModelOptions, RAGDBOptions, Protocol): + project_root: str + main_contract: str + system_doc: str + max_concurrent: int + cloud: bool # ← prover-only: run jobs in the cloud + ... +``` + +```python +# composer/foundry/entry.py +class FoundryArgs(TieredModelOptions, FoundryRAGDBOptions, Protocol): + project_root: str + main_contract: str + system_doc: str + forge_binary: str # ← foundry-only + forge_timeout_s: int # ← foundry-only + max_forge_runners: int # ← foundry-only + ... +``` + +Convention points worth naming: + +- **Foundry validates its precondition in the entry point** (`foundry.toml` must + exist) — application-specific input validation belongs here, before any service is + opened. +- **Each application owns its RAG DB choice.** Foundry overrides `--rag-db`'s default + to the cheatcodes DB via a Protocol (`FoundryRAGDBOptions`) rather than a new flag. +- **Run-close artifacts land in an `at_exit` hook** passed to `cli_pipeline` (both apps dump + `token_usage.json` there). `cli_pipeline` calls it from its own `finally` and swallows what it + raises, so a diagnostics failure can't mask the run's own outcome. +- **The entry point never imports a frontend.** It yields the Executor; `main()` + chooses the frontend. That is what lets one entry point back both a TUI and a + console `main()`. + +--- + +## 6. Piece 3 — the pipeline and its backend + +The application builds a `PipelineBackend` and hands it, with its env and its ecosystem, to +`cli_pipeline`'s continuation. The continuation is where the `PipelineRun` is assembled and the +shared driver [`run_pipeline`](../composer/pipeline/core.py) is called — so no application writes +that wiring: + +```python +# composer/pipeline/cli.py — the Continuation yielded by cli_pipeline +async def cont(env, backend, ecosystem) -> CorePipelineResult[FormT]: + run = PipelineRun( + ctx=full_ctx, source=full_source, env=env, + _semaphore=semaphore, _handler_factory=task_handler, + ) + return await run_pipeline( + backend=backend, run=run, ecosystem=ecosystem, + interactive=args.interactive, max_bug_rounds=args.max_bug_rounds, + threat_model=threat_model, + ) +``` + +```python +# each application supplies only the backend (composer/foundry/pipeline.py) +def backend(*, forge_binary, forge_timeout_s, source_input, forge_concurrency) -> FoundryBackend: + return FoundryBackend(FoundryArtifactStore(source_input.project_root), + _ForgeRunConfig(forge_binary, forge_timeout_s, ...)) +``` + +Two things to notice. The `handler_factory` (the frontend seam) and the run's two concurrency +semaphores are bundled into the `PipelineRun` — `run.runner(task_info, job)` is how every phase of +the driver spins up a task through whatever frontend was supplied, and `run.cpu_runner` is its peer +for a task that is *not* an agent (a toolchain build): same task machinery, charged to the CPU +budget (`--max-cpu-tasks`) rather than to the `--max-concurrent` agent slots. And the **ecosystem** is an explicit argument to the driver, not something +the backend carries: it supplies the analyzed-model type, the analysis prompts, `locate_main` and +the unit enumeration, so one backend shape can target more than one chain (see +[ecosystem-abstraction.md](./ecosystem-abstraction.md)). + +The backend itself is the four-slot contract the driver reads. The application maps +its phase enum onto the four **core phases** the driver tags: + +```python +# composer/spec/source/pipeline.py # composer/foundry/pipeline.py +core_phases = CorePhases({ core_phases = CorePhases({ + "analysis": AutoProvePhase.COMPONENT_ANALYSIS, "analysis": FoundryPhase.SYSTEM_ANALYSIS, + "extraction": AutoProvePhase.BUG_ANALYSIS, "extraction": FoundryPhase.PROPERTY_EXTRACTION, + "formalization": AutoProvePhase.CVL_GEN, "formalization": FoundryPhase.TEST_GENERATION, + "report": AutoProvePhase.REPORT, "report": FoundryPhase.REPORT, +}) }) +``` + +Everything below this — `preflight`, `prepare_system`, `prepare_formalization`, `formalize`, +`fetch_verdicts` — is the **formalization abstraction**, documented in full in +[formalization-abstraction.md](./formalization-abstraction.md). The one-line summary +of the contrast: + +| | autoprove (`ProverBackend`) | foundry (`FoundryBackend`) | +|---|---|---| +| `FormT` | `GeneratedCVL` | `GeneratedFoundryTest` | +| `preflight` | none | none (a building backend — Crucible — puts its build here, overlapping analysis) | +| `prepare_system` | harness lift + build prover tool | identity (`main_instance` only) | +| `prepare_formalization` | AutoSetup ∥ summaries ∥ invariants fan-out | trivial (formalizer already built) | +| `formalize` | author CVL, run prover, revise on CEX | author `.t.sol`, run `forge test` | +| `backend_guidance` | `CERTORA_BACKEND_GUIDANCE` | `FOUNDRY_BACKEND_GUIDANCE` | + +`backend_guidance` deserves a note as an application-shaping convention: it is a prose +string injected into the property-extraction prompt telling the agent what the +verification surface can and can't express. Foundry's, for instance, explains that a +fuzzer can't *prove* universals but *refutations are valuable* — so the same shared +extraction step produces backend-appropriate properties without the driver knowing +anything about it. + +--- + +## 7. Piece 4 — the frontend + +The frontend implements the `HandlerFactory` seam. The TUI frontends are thin +subclasses of the generic [`MultiJobApp[P, T]`](../composer/ui/multi_job_app.py) +(see [its design doc](../composer/ui/MULTI_JOB_DESIGN.md)). A frontend supplies four +things and inherits everything else: + +1. **Phase labels + section order** (constructor args), covered in §4. +2. **A per-task handler** — `create_task_handler`, returning a + `MultiJobTaskHandler` subclass. +3. **A per-task event handler** — `create_event_handler`, for domain-specific + streaming events beyond LLM messages. +4. **`make_handler`** — inherited from `MultiJobApp`; this *is* the `HandlerFactory`. + It mounts the panel/summary-row and calls the two `create_*` hooks. + +The autoprove and foundry TUIs are nearly identical in shape; they differ only in +what streams into a task's log. Both make their task handler double as its own +`EventHandler` via the `NullEventHandler` mixin: + +```python +# composer/ui/foundry_app.py +class FoundryTaskHandler(MultiJobTaskHandler[None], NullEventHandler): + async def handle_event(self, payload, path, checkpoint_id) -> None: + evt = cast(ForgeTestRunEvent, payload) + if evt["type"] == "forge_test_run": # stream forge run summaries + log = await self._ensure_forge_log() + log.write(evt["summary"]) + +class FoundryApp(MultiJobApp[FoundryPhase, FoundryTaskHandler]): + def __init__(self): + super().__init__(phase_labels=FOUNDRY_PHASE_LABELS, + section_order=FOUNDRY_SECTION_ORDER, + header_text="Foundry Test Author | ...") + def create_task_handler(self, panel, info) -> FoundryTaskHandler: + return FoundryTaskHandler(info.task_id, info.label, panel, self, ToolDisplayConfig()) + def create_event_handler(self, handler, info) -> EventHandler: + return handler # handler is its own event handler +``` + +```python +# composer/ui/autoprove_app.py — same structure; the domain events differ +class AutoProveTaskHandler(MultiJobTaskHandler[None], NullEventHandler): + async def handle_event(self, payload, path, checkpoint_id) -> None: + evt = cast(AutoProveEvent, payload) + match evt["type"]: + case "prover_output": ... # stream Certora Prover output lines + case "cloud_polling": ... # stream cloud job status +``` + +The autoprove handler additionally implements `handle_progress_event` to stream the +AutoSetup agent's output — an example of an application surfacing a backend-specific +sub-agent in its own panel. Neither handler supports HITL, so both raise from +`format_hitl_prompt` — a deliberate, explicit opt-out of a base-class hook. + +**The console frontend is the proof the seam works.** `AutoProveConsoleHandler` is a +*different* implementation of the same `HandlerFactory[AutoProvePhase, None]` that +renders to stdout instead of a Textual app: + +```python +# composer/ui/autoprove_console.py +class AutoProveConsoleHandler(MultiJobConsoleHandler[AutoProvePhase]): + """IOHandler[Never] + HandlerFactory for the auto-prove pipeline.""" +``` + +The pipeline can't tell the difference — it only ever calls `make_handler`. + +--- + +## 8. Piece 5 — the artifact store + +Deliverables are written through an [`ArtifactStore[I, FormT]`](../composer/spec/artifacts.py) +subclass — one per application. The base owns everything identical across +applications (`properties.json`, `commentary.md`, the property→units map, +`token_usage.json`); the subclass fixes the on-disk layout and adds the +format-specific bundle. This is covered in detail in +[formalization-abstraction.md §6](./formalization-abstraction.md); the application-level +point is the *convention that both applications share a project root without +colliding*: + +``` +autoprove → certora/specs/… certora/confs/… certora/ap_report/… +foundry → /*.t.sol certora/foundry/… certora/foundry/reports/… +``` + +Foundry deliberately materializes its `.t.sol` into the foundry project's own `test/` +dir (so `forge` finds them) but keeps all metadata under `certora/foundry/`, so a +co-located autoprove run and foundry run share one project without clobbering each +other's outputs. + +--- + +## 9. Piece 0 — the wiring: `main()` + +The `main()` in each `composer/cli/*.py` is the whole application in ~20 lines. It is +the *only* place that names both an entry point and a frontend, and its job is to +glue them via the seam and translate the result into user-facing output. + +```python +# composer/cli/tui_foundry.py (tui_autoprove.py is identical in shape) +async def _main() -> int: + summary = RunSummary() + async with _entry_point(summary) as pipeline: # piece 2: Executor + app = FoundryApp() # piece 4: frontend + async def work(): + result = await pipeline(app.make_handler) # ← the seam + app.notify(f"Foundry tests complete: {result.n_components} components, ...") + app.mark_pipeline_done() + app.set_work(work) + await app.run_async() # TUI owns the event loop + print(summary.format()) + return 0 +``` + +Two `main()` shapes exist, differing only in who owns the event loop: + +- **TUI** — the pipeline runs as a background *worker* inside the Textual app + (`app.set_work(work); await app.run_async()`), so the UI stays responsive while + the pipeline streams into it. +- **Console** — the pipeline runs directly and results print on completion: + + ```python + # composer/cli/console_autoprove.py + async with _entry_point(summary) as run: + result = await run(AutoProveConsoleHandler().make_handler) + print(summary.format()) + print(f" Components: {result.n_components} Properties: {result.n_properties}") + ``` + +Both call `import composer.bind as _` first — the side-effecting binding module that +must load before anything touches the DI container. + +--- + +## 10. Extending: defining a new application + +Because each piece is an implementation of a shared abstraction, adding an +application is a fill-in-the-blanks exercise; nothing in the driver, the UI base, or +the seam changes. + +1. **Phase enum** `P(enum.Enum)` — your task-grouping vocabulary, with the four core phases + (analysis / extraction / formalization / report) representable, plus a member to group + design-doc discovery under. +2. **Backend** — implement `PipelineBackend`, naming its eight type arguments, and its phase + objects (`preflight` → `prepare_system` → `PreparedSystem.prepare_formalization` → + `Formalizer`, or a `StagedFormalizer` when every unit shares one artifact), plus + `backend_guidance`, `core_phases`, `analysis_spec`, `artifact_store`, `to_artifact_id`. (Full + checklist in [formalization-abstraction.md §9](./formalization-abstraction.md).) +3. **Artifact store** — subclass `ArtifactStore`; define an `ArtifactIdentifier`. +4. **Result type** `FormT` satisfying `FormalResult` + `ReportableResult`. +5. **Entry point** — an `_entry_point` context manager that parses args (declared as a `Protocol` + extending `PipelineArgs`) and yields a `runner(handler)` closure which opens `cli_pipeline`, + builds the env + backend from the `StagedPipeline`, and calls the continuation with its + ecosystem. There is no per-application pipeline wrapper to write. +6. **Frontend(s)** — a `MultiJobApp[P, T]` subclass (phase labels, section order, + `create_task_handler`, per-task event streaming) and/or a console handler. +7. **`main()`** — glue an entry point to a frontend via `run(app.make_handler)`. + +The dependency direction is the guardrail: `main` → (entry point, frontend); +entry point → `cli_pipeline` → driver; driver → backend + `PipelineRun(handler_factory)`. Frontend +and backend never reference each other, and neither references `main`. Keep those +edges and the pieces stay swappable. + +For a Rust-wheel backend, steps 1–7 are all *declared* instead of written; see +[rust-applications.md](./rust-applications.md). + +--- + +## 11. Key files + +| Piece | autoprove | foundry | shared abstraction | +|---|---|---|---| +| Phase enum | [autoprove_app.py](../composer/ui/autoprove_app.py) | [foundry/pipeline.py](../composer/foundry/pipeline.py) | `HasName` ([multi_job.py](../composer/io/multi_job.py)) | +| Entry point / Executor | [autoprove_common.py](../composer/spec/source/autoprove_common.py) | [foundry/entry.py](../composer/foundry/entry.py) | `cli_pipeline` ([pipeline/cli.py](../composer/pipeline/cli.py)) | +| Backend | [spec/source/pipeline.py](../composer/spec/source/pipeline.py) | [foundry/pipeline.py](../composer/foundry/pipeline.py) | [pipeline/core.py](../composer/pipeline/core.py) · [pipeline/ptypes.py](../composer/pipeline/ptypes.py) | +| Frontend (TUI) | [autoprove_app.py](../composer/ui/autoprove_app.py) | [foundry_app.py](../composer/ui/foundry_app.py) | [multi_job_app.py](../composer/ui/multi_job_app.py) | +| Frontend (console) | [autoprove_console.py](../composer/ui/autoprove_console.py) | [foundry_console.py](../composer/ui/foundry_console.py) | [multi_console_handler.py](../composer/ui/multi_console_handler.py) | +| Artifact store | [spec/source/artifacts.py](../composer/spec/source/artifacts.py) | [foundry/artifacts.py](../composer/foundry/artifacts.py) | [spec/artifacts.py](../composer/spec/artifacts.py) | +| The seam | — | — | `HandlerFactory` / `TaskInfo` / `TaskHandle` ([multi_job.py](../composer/io/multi_job.py)) | +| `main()` | [tui_autoprove.py](../composer/cli/tui_autoprove.py) · [console_autoprove.py](../composer/cli/console_autoprove.py) | [tui_foundry.py](../composer/cli/tui_foundry.py) · [console_foundry.py](../composer/cli/console_foundry.py) | — | +| All five, declared | — | — | [composer/rustapp/](../composer/rustapp/) ([doc](./rust-applications.md)) | diff --git a/docs/command-sandbox.md b/docs/command-sandbox.md index 21fc92d6..04c4cd47 100644 --- a/docs/command-sandbox.md +++ b/docs/command-sandbox.md @@ -1,40 +1,41 @@ -# Design — Sandboxing the `RunCommand` effect (Phase 6) - -**Status:** implemented. Design + record for [crucible-application.md §7.4](./crucible-application.md#L436) -and [§9 Phase 6](./crucible-application.md#L634) — the *required*, definition-of-done phase. The -sandbox mechanism is built and validated (§9 steps 1–5 done, gate §10 green — incl. the full LLM -e2e passing under the launcher); Crucible runs confined by default. Open items are orthogonal to the -sandbox (§11): a shared-`Cargo.toml` feature race that lost one of three instructions, and per-run -`CARGO_HOME`/tightening follow-ups. - -**One-line summary.** Every command run through the `RunCommand` effect compiles and/or runs -LLM-authored *native* code (§7.2). Today that runs with the full ambient environment of the -AutoProver process. Phase 6 confines each such command — with no network, no inherited secrets, and +# Design — Sandboxing untrusted command execution + +**Status:** implemented. The mechanism lives in this repo — [`composer/sandbox/`](../composer/sandbox/) +(policy, provider seam, launcher mapping, recipes) plus the +[`run-confined`](../rust/run-confined) launcher binary — and is validated by the escape suite +(§10 A). It was built for, and first consumed by, the Crucible backend; **Crucible itself and the +Solana build step now live outside this repository**, so the run-level gates that exercised the +legitimate path (§10 B, the LLM e2e) run there, and the per-consumer notes below say which side +each piece is on. Open items are in §11. + +**One-line summary.** Every toolchain command a formalization backend runs compiles and/or runs +LLM-authored *native* code (§2). Unconfined, that runs with the full ambient environment of the +AutoProver process. This confines each such command — no network, no inherited secrets, and only its own inputs on the filesystem — using **unprivileged, in-process kernel sandboxing (Landlock + seccomp)** that needs no container changes, no namespaces, no capabilities, and no -custom runtime. It is a single wrapper around [`run_local_command`](../composer/sandbox/command.py). -Done is proven by an escape test. +custom runtime. One authored policy serves both launch paths (§4). Done is proven by an escape test. --- ## 1. Why this is required, not optional The outer AutoProver container protects the *host* from AutoProver. It does **not** protect -AutoProver's own secrets, network access, and filesystem from code running *inside* it. And the -`RunCommand` effect deliberately runs untrusted native code: +AutoProver's own secrets, network access, and filesystem from code running *inside* it. And a +backend's toolchain steps deliberately run untrusted native code: - `cargo build-sbf` on the **user-supplied program** compiles it natively — running its `build.rs`, its proc-macros, and (for a future Prover/CVLR backend) LLM-munged source. - `crucible run` compiles the **LLM-authored harness** (its `setup()`, `action_*`, `build.rs`) - and then runs it as a native LiteSVM-in-process binary (§7.2 — verified native, no SVM sandbox). + and then runs it as a native LiteSVM-in-process binary (verified native — there is no SVM sandbox + around it). So arbitrary code of the LLM's (and the analyzed program's) choosing executes with whatever ambient authority the AutoProver process has. -The trust boundary from §7.2 ("the LLM authors only file *contents*, never argv") stops the LLM -from choosing *what command runs* — it does nothing about what that command, once running, can -*reach*. That is this phase's job. +The standing trust boundary — **the LLM authors only file *contents*, never argv** +([rust-applications.md §8](./rust-applications.md)) — stops the LLM from choosing *what command +runs*. It does nothing about what that command, once running, can *reach*. That is the sandbox's job. -Until this phase is green the backend may run only in a trusted, offline environment on trusted +Without the sandbox a backend may run only in a trusted, offline environment on trusted input (the gate scenario). This is the definition of done. --- @@ -45,9 +46,9 @@ input (the gate scenario). This is the definition of done. |---|---| | **Asset** | AutoProver's ambient secrets, and host files outside the command's declared inputs. | | **Adversary** | Native code the LLM authored (harness `setup`/`action`/`build.rs`) **and** native code in the analyzed program (its `build.rs`, proc-macros) that `cargo build-sbf` runs. Assume it is actively hostile and knows it is being fuzzed. | -| **Trust boundary** | The process boundary of each `RunCommand` invocation. Inside: untrusted. Outside: the trusted AutoProver process. `program`+`args` are trusted (Rust decider / Python build step author them, §7.2); only the *files* are untrusted. | -| **Assumptions** | (1) The outer container/host is the infrastructure's boundary against the host machine and other tenants (on EC2, the Nitro hypervisor) — this phase is the boundary *within* the container, between AutoProver and its own untrusted child. (2) The kernel is patched and Landlock-capable (§8). (3) The host toolchains we grant read access are trusted. | -| **Non-goals** | Protecting the host machine *from the container* (the infrastructure does that). A full VM boundary between AutoProver and the child (that is what gVisor/Kata/VM-per-run would add at the infra layer, orthogonal to this phase — §6). Defending against a malicious *`program`/`args`* — those are trusted by construction (§7.2). | +| **Trust boundary** | The process boundary of each confined command. Inside: untrusted. Outside: the trusted AutoProver process. `program`+`args` are trusted — the compiled wheel or a trusted Python build step authors them; only the *files* are untrusted. | +| **Assumptions** | (1) The outer container/host is the infrastructure's boundary against the host machine and other tenants (on EC2, the Nitro hypervisor) — the sandbox is the boundary *within* the container, between AutoProver and its own untrusted child. (2) The kernel is patched and Landlock-capable (§8). (3) The host toolchains we grant read access are trusted. | +| **Non-goals** | Protecting the host machine *from the container* (the infrastructure does that). A full VM boundary between AutoProver and the child (that is what gVisor/Kata/VM-per-run would add at the infra layer, orthogonal to the sandbox — §6). Defending against a malicious *`program`/`args`* — those are trusted by construction. | **Explicit guarantees the sandbox must provide:** @@ -73,7 +74,7 @@ their real needs: | Command | Reads (grant **ro+x**) | Writes (grant **rw**) | Network | |---|---|---|---| | `cargo build-sbf ` | rust toolchain (`RUSTUP_HOME`), solana platform-tools (the sBPF toolchain), warm cargo registry (`CARGO_HOME/registry`), program crate source | program crate `target/` | none (offline) | -| `crucible run …` | the `crucible` binary + its libs, rust toolchain, cargo registry, the **crucible checkout crates** (path deps from `CrucibleDep`, §6.1), the built `.so` + IDL | the harness crate `target/`, corpus/output dirs | none (offline) | +| `crucible run …` | the `crucible` binary + its libs, rust toolchain, cargo registry, the **checker's checkout crates** (the path deps the wheel's manifest names), the built `.so` + IDL | the harness crate `target/`, corpus/output dirs | none (offline) | | `cargo build` (harness, if run directly) | as above | harness `target/` | none (offline) | Common surface, resolved once at sandbox-config time and expressed as Landlock rules (§6): @@ -106,20 +107,33 @@ Common surface, resolved once at sandbox-config time and expressed as Landlock r Everything else — the rest of the bind-mounted project, `/etc`, `/proc/`, `$HOME`, the process environment — is **not granted**, therefore inaccessible. Confinement is default-deny. -> The exact host paths (`RUSTUP_HOME`, platform-tools dir, crucible binary) are **resolved by the -> host at config time**, not hardcoded — see the `SandboxPolicy` in §7. They are discovered from the -> environment the same way `resolve_crucible_repo` already discovers the checkout. +> The exact host paths (`RUSTUP_HOME`, platform-tools dir, a checker's binary) are **resolved at +> config time**, not hardcoded — see the `SandboxPolicy` in §7. The generic ones are discovered from +> the environment by the `rust_build_policy` recipe; the ones only a particular backend knows about +> (its checker's checkout and binary dir) are contributed by that backend as `extra_ro` — for a Rust +> wheel, through its pure `sandbox_grants` callout, so the wheel *declares* grants and Python still +> decides the policy. --- -## 4. The seam — one function, unchanged signature +## 4. The seam — one policy, two launch paths -All command execution already funnels through -[`run_local_command`](../composer/sandbox/command.py) (both the IoC `RunCommand` effect via -[`RealEffects.run_command`](../composer/rustapp/adapter.py#L120) and the Solana build step -[`build_program`](../composer/spec/solana/build.py)). It lives in the backend-agnostic -[`composer/sandbox`](../composer/sandbox/) package — outside `rustapp` — so Python-based backends can -run confined commands too, not just the Rust-IoC ones. The sandbox wraps exactly this one function. +Command execution funnels through one of two launch paths, and **both consume the same +`SandboxPolicy`**: + +- [`run_local_command`](../composer/sandbox/command.py) — the Python runner, used by trusted Python + build steps (the Solana sBPF build / IDL step, now behind the + [`WorkspaceToolchain`](../composer/rustapp/toolchain.py) seam). It lives in the backend-agnostic + [`composer/sandbox`](../composer/sandbox/) package — outside `rustapp` — so Python-based backends + can run confined commands too. +- A **Rust wheel's own `compile`/`validate`**, which spawn the launcher directly via + `autoprover_sdk::sandbox::Workspace::run` rather than calling back into Python. They receive the + policy already lowered to an opaque argv prefix (`SandboxConfig.backend_spec` → + `LauncherProvider.argv_prefix`) and simply prepend it — see + [rust-applications.md §8](./rust-applications.md). + +The sandbox wraps exactly these, and the policy/provider seam below is what keeps the two in step: +one authored intent, two launchers. **The mechanism sits behind a `SandboxProvider` seam, so it is swappable.** `run_local_command` never names a concrete tool. It holds a **tool-agnostic `SandboxPolicy`** (the *intent*: rw paths, @@ -137,16 +151,18 @@ create_subprocess_exec(*spec.argv, cwd=workdir, env=spec.env, …) ``` The first provider is our **custom launcher shim** (§6): `LaunchSpec.argv == ["run-confined", -*policy_argv, "--", program, *args]`, all authored by trusted Python (never the LLM). Swapping to an -off-the-shelf tool later — `landrun`, `sandlock` — is a *new `SandboxProvider` implementation that -maps the same `SandboxPolicy` to that tool's flags*; the policy, this seam, `run_local_command`, -`RealEffects`, and the escape-test gate (§10) are all untouched. The provider is chosen by config -(`CommandConfig` / an env var), defaulting to the custom launcher. The `none` provider is a -passthrough (`argv == [program, *args]`) — byte-for-byte today's behavior for the EVM/Foundry paths -and explicit trusted-input dev runs. - -Nothing in the Rust decider, the ABI, the driver, or the artifact store changes — this is why §7.4 -could defer it to last. +*policy_argv, "--", program, *args]`, all authored by trusted Python (never the LLM). The same +provider also exposes that wrapper on its own, as `argv_prefix(policy)` — everything *except* the +`program args` — which is what lets a Rust wheel launch a confined command without Python in the +loop. Swapping to an off-the-shelf tool later — `landrun`, `sandlock` — is a *new `SandboxProvider` +implementation that maps the same `SandboxPolicy` to that tool's flags*; the policy, this seam, +`run_local_command`, the wheel side, and the escape suite (§10) are all untouched. The provider is +chosen by [`SandboxConfig`](../composer/sandbox/config.py) (`$COMPOSER_SANDBOX_PROVIDER`). The `none` +provider is a passthrough (`argv == [program, *args]`, and an **empty** `argv_prefix`) — byte-for-byte +the unconfined behavior, for the EVM/Foundry paths and explicit trusted-input dev runs. + +Nothing in the backend ABI, the driver, or the artifact store changes — which is why confinement +could be added last, and why a wheel names no sandbox mechanism anywhere. Two properties `run_local_command` *already* enforces stay in force and are the first line of defense (the sandbox is the second): the command runs via **exec, not a shell**, and every written @@ -167,22 +183,24 @@ splits cleanly along the code-execution line: - **`cargo build` runs build scripts and proc-macros** — this is where untrusted code executes, so it happens **inside** the sandbox, `--offline`, against the already-warm cache. -The harness `Cargo.toml` is **host-owned** (`CrucibleDep.render_deps`, pinned versions, §6.1), so +The harness `Cargo.toml` is **authored by the trusted wheel** (pinned versions, never LLM text), so its dep graph is fixed and vendorable deterministically. The program-under-test's `Cargo.toml` is user-supplied, but `cargo fetch` on it is still exec-free, so the same split holds for the build-sbf step. This also closes the build-time supply-chain vector: with offline + a pre-warmed cache, a malicious `build.rs` cannot pull a payload at build time. -**Implementation (step 4).** "Offline inside" is one env var, not per-tool flags: the policy sets -**`CARGO_NET_OFFLINE=1`** in the child env, which forces *every* cargo invocation offline — including -the nested `cargo` that `crucible run` spawns to build the harness — so we never thread `--offline` -through each tool ([recipes.py](../composer/sandbox/recipes.py), `offline=True` default). "Fetch -outside" is [`warm_cargo_cache`](../composer/spec/solana/build.py) — a `cargo fetch` run *unsandboxed* -(no provider → network on) before the confined build; `build_program` calls it before the sandboxed -`cargo build-sbf`. The harness crate has its own deps (libafl, litesvm, …), so it needs its own warm -at manifest-assembly time; wiring that exact call site (and confirming whether `CARGO_HOME` must be -granted rw for cargo's build-time source extraction, or pre-extracted during the warm) lands with the -gate in step 5, where a real offline build proves it. All of this is inert until a sandbox is enabled. +**Implementation.** "Offline inside" is one env var, not per-tool flags: the policy sets +**`CARGO_NET_OFFLINE=true`** in the child env, which forces *every* cargo invocation offline — +including a nested `cargo` that a checker spawns to build a harness — so we never thread `--offline` +through each tool ([recipes.py](../composer/sandbox/recipes.py), `offline=True` default). The value +must be exactly `true`: cargo parses it as a config boolean and rejects anything else, so a truthy +`1` aborts the build *and* leaves it online. "Fetch outside" is a `cargo fetch` run *unsandboxed* +(no provider → network on) before the confined build. +Both halves of that prep are now **declared, not called**: a wheel's `workspace_prep` names the dirs +to warm and the program to build, and the chain's registered `WorkspaceToolchain` performs them — +fetch unconfined, build confined + offline (see +[rust-applications.md §7](./rust-applications.md)). That keeps the network posture Python-owned while +the wheel supplies no command line. All of it is inert until a sandbox is enabled. --- @@ -208,7 +226,7 @@ host-kernel attack surface across *all* of AutoProver — the opposite of what a should do) or running AutoProver under a **gVisor/Kata** runtime. gVisor works, but (a) it imposes its *heaviest* overhead precisely on our syscall/I/O-bound compile+fuzz workload, and (b) its benefit — protecting the host kernel — is an *infrastructure* boundary that on EC2 is already provided by the -Nitro hypervisor. Neither is worth coupling this phase to a deployment decision. +Nitro hypervisor. Neither is worth coupling the sandbox to a deployment decision. ### The chosen model: the process sandboxes itself @@ -355,12 +373,16 @@ class SandboxPolicy: # program + args come per-call from run_local_command ``` -**Provider selection is separate config, not part of the policy** — a `CommandConfig.sandbox_provider` -knob (`"launcher"` = the custom Rust shim, default; `"none"` = passthrough; later `"landrun"` / -`"sandlock"`), overridable by env var. `run_local_command` gains `policy: SandboxPolicy | None` + -the resolved provider (default provider `"none"` when no policy, so existing callers and the EVM path -are unchanged). `RealEffects` builds the policy from a host-resolved config (toolchain paths -discovered like `resolve_crucible_repo` already does), and `build_program` uses the same. +**Provider selection is separate config, not part of the policy** — +[`SandboxConfig`](../composer/sandbox/config.py) carries the provider name (`"launcher"` = the custom +Rust shim; `"none"` = passthrough, the default; later `"landrun"` / `"sandlock"`), overridable by +`$COMPOSER_SANDBOX_PROVIDER`, plus the `extra_ro` / `env_passthrough` a consumer adds. It builds the +policy for a workdir (`build_policy`) for the Python path and lowers it to an argv prefix +(`backend_spec`) for the wheel path. A Rust application declares that it wants confinement +(`confine_by_default`) and contributes its extra grants (`sandbox_grants`); the host, never the wheel, +constructs the config — see [rust-applications.md §8](./rust-applications.md). `run_local_command` +takes `policy: SandboxPolicy | None` + the resolved provider (no policy → `"none"`, so existing +callers and the EVM path are unchanged). **Fail-closed.** Before running under a real sandbox provider, `provider.available()` is checked (for the launcher: Landlock is present *and* actually enforcing). If it isn't — or the provider cannot apply its @@ -411,65 +433,68 @@ the sandbox is unavailable: refuse to run, loudly, rather than run untrusted nat → fail-closed (§7). Enforcement smoke-tested on the host (write-outside / planted host file / `/proc//environ` / inet+io_uring+netlink sockets all denied; workdir write, AF_UNIX, and toolchain `exec` allowed); argv mapping golden-tested. Full escape gate is step 5. -3. **Thread `policy` + provider through `run_local_command`** — *done*: the runner accepts - `provider`/`policy` (default `None` → the `none` passthrough, byte-for-byte today's behavior) and - is fail-closed via `ensure_available`. A `SandboxConfig` ([composer/sandbox/config.py](../composer/sandbox/config.py)) +3. **Thread `policy` + provider through both launch paths** — *done*: `run_local_command` accepts + `provider`/`policy` (default `None` → the `none` passthrough, byte-for-byte the unconfined + behavior) and is fail-closed via `ensure_available`. A `SandboxConfig` + ([composer/sandbox/config.py](../composer/sandbox/config.py)) selects the provider (`$COMPOSER_SANDBOX_PROVIDER`, default `none`) and builds the policy via the `rust_build_policy` recipe ([composer/sandbox/recipes.py](../composer/sandbox/recipes.py) — the workdir and `/dev` nodes rw; discovered rust/cargo/platform-tool and system dirs ro, incl. `/etc` - for NSS; env allowlist; network off). Threaded through `RealEffects` and `RustBackend`/`RustFormalizer` - ([composer/rustapp/adapter.py](../composer/rustapp/adapter.py)), `build_program` - ([composer/spec/solana/build.py](../composer/spec/solana/build.py)), and the Crucible pipeline - (which adds the crucible checkout + binary to `extra_ro`). Integration-tested: `run_local_command` - under the launcher denies out-of-workdir reads and network while allowing the workdir + toolchain. -4. **Offline prep (§5)** — *done*: `warm_cargo_cache` (a `cargo fetch` run outside the sandbox, - network on) warms the registry, and the policy sets `CARGO_NET_OFFLINE=1` so the confined build — - and the nested cargo `crucible run` spawns — run offline. Wired into `build_program`; the - harness-dir warm is `CrucibleArtifactStore.warm_dependencies`, called from `prepare_formalization` - after the manifest is placed when a sandbox is on. `CARGO_HOME` is granted rw (the crucible policy) - so cargo can extract crate sources offline. -5. **The escape-test gate (§10)** — *done*, and **Crucible's default provider is now `launcher`** - (`_crucible_sandbox`; override with `COMPOSER_SANDBOX_PROVIDER=none`). Validated: + for NSS; env allowlist; network off). The wheel path gets the same policy as an opaque + `argv_prefix` through `backend_spec`, threaded by `RustBackend`/`RustFormalizer` + ([composer/rustapp/adapter.py](../composer/rustapp/adapter.py)); a wheel's `sandbox_grants` is what + adds a checker's own read-only paths (a tool checkout, its binary dir) to `extra_ro`. + Integration-tested: `run_local_command` under the launcher denies out-of-workdir reads and network + while allowing the workdir + toolchain. +4. **Offline prep (§5)** — *done*: a `cargo fetch` run outside the sandbox (network on) warms the + registry, and the policy sets `CARGO_NET_OFFLINE=true` so the confined build — and any nested cargo a + checker spawns — run offline. Both are now declared by the wheel's `workspace_prep` and performed + by the chain's `WorkspaceToolchain` (§5). `CARGO_HOME` is granted rw (pointed at the private + per-run home, §11 item 5) so cargo can extract crate sources offline. +5. **The escape suite (§10 A)** — *done*, and a wheel that declares `confine_by_default` gets the + `launcher` provider by default (override with `COMPOSER_SANDBOX_PROVIDER=none`). Validated: - **Part A (escape suite) — green** ([tests/test_sandbox_escape.py](../tests/test_sandbox_escape.py)): a `rustc`-compiled malicious program run through the real launcher has every vector *denied* (secret env, `/proc//environ`, host file outside the workdir, external TCP, and `169.254.169.254`), with an unconfined control confirming the leaks would otherwise happen. - - **Part B — green**: a real `cargo-build-sbf` of `solana_vault` under the launcher (offline, - confined) produces the `.so` ([tests/test_crucible_sandbox_gate.py](../tests/test_crucible_sandbox_gate.py) - — this caught the relative-policy-path bug; grants must be absolute), and a real - `crucible run --dry-run` under the launcher builds the harness *offline* and runs LiteSVM - (`Harness validation passed!`). - - **Full LLM vertical — green**: the e2e gate (`tests/test_crucible_e2e_gate.py`) passes under the - launcher (`COMPOSER_SANDBOX_PROVIDER=launcher`): analysis → 23 properties → shared fixture - authored → per-instruction harness build + fuzz, all confined + offline, with **all three - instructions (initialize / deposit / withdraw) delivered with fuzz verdicts** (`BAD` — - counterexamples found). Getting here required the `/tmp` fix below and the shared-crate - concurrency fix (§11 item 8); before the latter, `initialize` was dropped to a `Cargo.toml` - feature race. - - **Root cause found via the gate:** every fresh harness build initially failed at the *link* step — + - **Part B (the legitimate path) — green when it was validated, and now lives with the + consumer**: a real `cargo-build-sbf` of a Solana program under the launcher (offline, confined) + produced the `.so` — this is what caught the relative-policy-path bug, so grants must be + absolute — and a real checker dry-run under the launcher built the harness *offline* and ran + LiteSVM. Those gates moved out with the Crucible backend; nothing in this repo exercises a real + toolchain build under the launcher, so a change to the *grants* (§3) can only be re-validated + against a consumer. + - **Full LLM vertical — green** at the time, under `COMPOSER_SANDBOX_PROVIDER=launcher`: analysis + → properties → shared fixture authored → per-unit harness build + fuzz, all confined + offline, + with every unit delivered with fuzz verdicts. Getting there required the `/tmp` fix below and the + shared-crate concurrency fix (§11 item 8). + + **Root cause found via that gate:** every fresh harness build initially failed at the *link* step — `Cannot create temporary file in /tmp/: Permission denied` (the linker's `$TMPDIR` scratch, which the policy didn't grant). A link failure reads as "could not compile", so the LLM kept rewriting a fine fixture. Fixed by redirecting `TMPDIR` to a private `/.sandbox_tmp` (§3) rather than - granting the shared `/tmp`. The `RunCommand` failure logging added alongside the authoring - improvements is what surfaced it. - -Each step is behind the seam, so the earlier Phase 1–5 gates keep passing. **Prerequisite of the -flip:** `run-confined` must be resolvable — `$RUN_CONFINED_BIN`, then PATH, then the dev build -(`cargo build -p run-confined --release`). Containers opt in via the `scripts/docker-compose.sandbox.yml` -overlay, which builds the launcher (`scripts/Dockerfile.sandbox`) and mounts it read-only at -`$RUN_CONFINED_BIN`. Otherwise Crucible fail-closes (§7/§8). A later off-the-shelf swap -(`landrun`/`sandlock`) is *only* a new step-2-style provider — the seam, policy, and gate are + granting the shared `/tmp`. Logging the failing command's output is what surfaced it. + +Each step is behind the seam, so every earlier gate kept passing. **Prerequisite of running +confined:** `run-confined` must be resolvable — `$RUN_CONFINED_BIN`, then PATH (a development +checkout gets it in `.venv/bin` from `uv sync`, which builds `rust/run-confined` as a bin +wheel). Containers opt in via the +`scripts/docker-compose.sandbox.yml` overlay, which builds the launcher +(`scripts/Dockerfile.sandbox`) and mounts it read-only at `$RUN_CONFINED_BIN`. Otherwise a +confine-by-default application fail-closes (§7/§8). A later off-the-shelf swap +(`landrun`/`sandlock`) is *only* a new step-2-style provider — the seam, policy, and escape suite are untouched. --- ## 10. The gate — an escape test -A new expensive gate (`tests/test_crucible_sandbox_gate.py`) with two halves: +Two halves. **A** lives here as [tests/test_sandbox_escape.py](../tests/test_sandbox_escape.py) and +runs against the real launcher; **B** exercises a real toolchain build and therefore lives with the +consuming application. -**A. Escape attempts are denied.** Author a *malicious* harness whose `setup()` (and a companion -`build.rs`) attempts, and record the outcome of each: +**A. Escape attempts are denied.** Compile a *malicious* program whose entry point (and a companion +`build.rs`) attempts each of the following, recording the outcome: - **Read a planted secret** — set a fake `ANTHROPIC_API_KEY=canary-` in AutoProver's env before the run; the harness (a) reads `std::env::var("ANTHROPIC_API_KEY")` and (b) tries to read @@ -486,19 +511,21 @@ A new expensive gate (`tests/test_crucible_sandbox_gate.py`) with two halves: - **Cargo credentials** — a planted `credentials.toml` under the shared cargo home is **not** readable (policy grants `bin/` only). -The harness must not be able to fail the assertions silently — it writes each probe's result into -the workdir (allowed) and the test reads them back, asserting every probe reports *denied*. +The program must not be able to fail the assertions silently — it writes each probe's result into +the workdir (allowed) and the test reads them back, asserting every probe reports *denied*. An +unconfined control run confirms the leaks would otherwise happen, so a vacuously-passing suite (the +program failing to run at all) can't read as success. -**B. The legitimate path still works.** The existing `solana_vault` gate ([§8](./crucible-application.md#L545)) -passes **unchanged** under the launcher provider — the shared fixture is authored, the `.so` builds, -tests compile and fuzz, verdicts are produced. This proves the sandbox grants exactly the toolchain -the real work needs and nothing more. +**B. The legitimate path still works.** A backend's own run-level gate passes **unchanged** under the +launcher provider — the shared artifact is authored, the program builds, units compile and are +checked, verdicts are produced. This is what proves the sandbox grants exactly the toolchain the real +work needs and nothing more, and it can only be run where a real backend + program live. Because the gate is written against the `SandboxProvider` seam (§4), not a specific tool, it doubles as the **conformance test any future provider must pass** — swapping in `landrun`/`sandlock` means re-running this same gate green, nothing more. -Only when both halves are green may the backend run on untrusted input (the §9 definition of done). +Only when both halves are green may a backend run on untrusted input (§1's definition of done). --- @@ -529,7 +556,7 @@ Only when both halves are green may the backend run on untrusted input (the §9 sources, takes locks), and that build runs untrusted `build.rs`/proc-macro code — so a writable *shared* `~/.cargo` was a cross-run poisoning surface (overwrite an extracted `registry/src` to hit a later run). Fixed: `rust_build_policy` points `CARGO_HOME` at a **private per-run dir under - the workdir** (`sandbox_cargo_home` → `/.sandbox_cargo`), the warm step (`warm_cargo_cache`, + the workdir** (`sandbox_cargo_home` → `/.sandbox_cargo`), the warm step (a `cargo fetch`, unsandboxed) fetches *into that same home*, and the shared cargo home is granted **read-only on `bin/` only** (`shared_cargo_ro_paths`) — never the home root, so `credentials.toml` cannot be read by untrusted code. Untrusted writes touch only the run's throwaway cache. Validated: a fresh @@ -546,13 +573,13 @@ Only when both halves are green may the backend run on untrusted input (the §9 deployments running genuinely untrusted programs should also apply the standard EC2 hardening — least-privilege instance IAM role, IMDSv2 with hop limit 1, egress-restricted security group, and (if desired) VM-per-run or a gVisor runtime. Decide per deployment when the tenancy model is - settled; none of it blocks Phase 6. -8. **Shared-`Cargo.toml`/`main.rs` race (crucible backend) — fixed.** The per-component sessions - share one `fuzz//` crate; concurrent runs raced on both files (the observed - "package does not contain this feature: `c_`" that dropped `initialize`, and a latent - `main.rs` clobber). Fixed two ways: `prepare_component` now reserves Cargo features - **cumulatively** (the manifest only grows, so no feature is lost), and per-component command runs - are **serialized + atomic** (`run_local_command` materializes files and runs as one unit under a - `Semaphore(1)` shared by `RustFormalizer`), while the LLM authoring turns still run concurrently. - The remaining parallelism win — concurrent *builds/fuzzing* — needs a crate-per-component (§10 Q1); - deferred. + settled; none of it blocks or is blocked by the in-process sandbox. +8. **Shared-crate race (a backend whose units share one crate) — fixed.** Per-unit runs against one + shared crate raced on its `Cargo.toml` and `main.rs` (the observed "package does not contain this + feature" that silently dropped a unit, plus a latent source clobber). The fix is now structural + rather than a mutation protocol: the wheel **materializes the manifest and source per confined + run** from that run's `files` map, so no two runs mutate a shared file, and a wheel that shares a + build dir declares `serialize_toolchain`, which puts its blocking callouts behind one + `Semaphore(1)` while the LLM authoring turns still run concurrently + ([rust-applications.md §3](./rust-applications.md)). The remaining parallelism win — concurrent + builds/checks — needs the wheel to split building from running; deferred. diff --git a/docs/ecosystem-abstraction.md b/docs/ecosystem-abstraction.md index 2533e59d..d65476fb 100644 --- a/docs/ecosystem-abstraction.md +++ b/docs/ecosystem-abstraction.md @@ -7,9 +7,9 @@ the source-reading conventions, connectivity validation, how the target's "main" and how it is split into units. Everything downstream of properties (how a property becomes a verified artifact) belongs to the **backend**, a separate axis. -Today two ecosystems are implemented: `EVM` (Solidity, fully wired to the CVL/prover and -Foundry backends) and `SOLANA` (Rust, front half only — analysis + property extraction, gated -by a null backend). +Today three ecosystems are implemented: `EVM` (Solidity, wired to the CVL/prover and Foundry +backends), `SOLANA` (Rust front half, gated by a null backend), and `SOROBAN` (Rust front half, +with no backend yet). --- @@ -50,14 +50,13 @@ not the language a backend is implemented in. ```python LanguageTag = Literal["solidity", "rust"] -ChainTag = Literal["evm", "solana", "soroban"] # "soroban" is reserved; not yet wired +ChainTag = Literal["evm", "solana", "soroban"] @dataclass(frozen=True) class Language: name: LanguageTag default_forbidden_read: str # fs-exclusion regex (Cargo layout vs Foundry layout) - code_explorer_prompt: str # source-navigation framing ("Rust source" vs "Solidity") - vulnerability_patterns_partial: str | None = None # j2 partial of language-level vulnerability patterns + vulnerability_patterns_fragment: str | None = None # j2 fragment of language-level vulnerability patterns @dataclass(frozen=True) class Ecosystem[App: BaseApplication, Main, Unit: FeatureUnit]: @@ -70,6 +69,7 @@ class Ecosystem[App: BaseApplication, Main, Unit: FeatureUnit]: locate_main: Callable[[App, SourceCode], Main] # find the "main" contract/program units: Callable[[Main], list[Unit]] # split into per-unit extraction items analysis_extra_input: Callable[[SourceCode], list[str | dict]] + code_explorer_prompt: TypedTemplate # shared protocol + this chain's look-fors ``` `Main` and `Unit` generalize what were EVM's `ContractInstance` / `ContractComponentInstance` — @@ -78,11 +78,16 @@ thin index wrappers over `App` that the driver hands to the backend and to prope interface (`display_name` / `slug` / `unit_index` / `cache_material` / `context_tag` / `feature_json`) the driver uses for per-unit cache keys, task ids, and labels. -A registry exposes the ecosystems by chain tag; it is heterogeneous in `App`/`Main`/`Unit` -(each chain has its own model), hence `Ecosystem[Any, Any, Any]`: +A registry exposes the ecosystems by chain tag. Each chain has its own `App`/`Main`/`Unit` types, +so the registry is a `TypedDict` rather than a plain `dict`. ```python -ECOSYSTEMS: dict[ChainTag, Ecosystem[Any, Any, Any]] = {"evm": EVM, "solana": SOLANA} +class Ecosystems(TypedDict): + evm: EvmEcosystem + solana: SolanaEcosystem + soroban: SorobanEcosystem + +ECOSYSTEMS: Ecosystems = {"evm": EVM, "solana": SOLANA, "soroban": SOROBAN} ``` --- @@ -97,7 +102,6 @@ backends run against it unchanged. SOLIDITY = Language( name="solidity", default_forbidden_read=fs_forbidden_read, # Foundry layout: lib/, test/, .sol carve-out - code_explorer_prompt=CODE_EXPLORER_SYS_PROMPT, ) EVM: Ecosystem[SourceApplication, ContractInstance, ContractComponentInstance] = Ecosystem( @@ -110,6 +114,7 @@ EVM: Ecosystem[SourceApplication, ContractInstance, ContractComponentInstance] = locate_main=main_instance, # match by solidity_identifier units=_evm_units, # one unit per contract component analysis_extra_input=_evm_analysis_extra_input, + code_explorer_prompt=EVM_CODE_EXPLORER_TEMPLATE, # code_explorer/solidity.j2 ) ``` @@ -129,8 +134,7 @@ RUST = Language( name="rust", # Cargo/Anchor layout: hide build output, VCS, lockfiles, and the JS side; keep crate sources + tests/. default_forbidden_read=r"(^target/.*)|(^\.git.*)|(^node_modules/.*)|(.*\.lock$)", - code_explorer_prompt=RUST_CODE_EXPLORER_PROMPT, # "Rust source … instruction handlers, Accounts, PDAs" - vulnerability_patterns_partial="rust/_vulnerability_patterns.j2", # overflow/underflow, panic!/unwrap/expect, ownership + vulnerability_patterns_fragment="rust/vulnerability_patterns_fragment.j2", # overflow/underflow, panic!/unwrap/expect, ownership ) SOLANA: Ecosystem[SolanaApplication, SolanaProgramInstance, SolanaComponentInstance] = Ecosystem( @@ -143,6 +147,7 @@ SOLANA: Ecosystem[SolanaApplication, SolanaProgramInstance, SolanaComponentInsta locate_main=_solana_locate_main, # match by program_identifier units=_solana_units, # one per ProgramComponent of the main program analysis_extra_input=_solana_analysis_extra_input, + code_explorer_prompt=SOLANA_CODE_EXPLORER_TEMPLATE, # rust/_common + PDAs / signers / CPI identity ) ``` @@ -176,23 +181,73 @@ SOLANA: Ecosystem[SolanaApplication, SolanaProgramInstance, SolanaComponentInsta The `RUST` language facet is chain-independent, so its source conventions and vulnerability-pattern fragment are authored once and pulled into the chain's prompts by Jinja `{% include %}`. The -Solana property template composes the shared Rust fragment with its own platform fragment: +code-explorer system prompt is the same split, but on the *ecosystem*: a shared protocol +(`code_explorer/common_fragment.j2`) plus a Rust crate-navigation fragment +(`code_explorer/rust/common_fragment.j2`) plus the chain's look-fors (`code_explorer/solana.j2` / +`code_explorer/soroban.j2`). A single Rust explorer prompt cannot name PDAs without lying to +Soroban, or `require_auth` without lying to Solana. + +The Solana property template composes the shared Rust fragment with its own platform fragment: ```jinja {# composer/templates/solana/property_prompt.j2 #} -{% include "rust/_vulnerability_patterns.j2" %} {# shared: overflow, panics, unwrap, lossy casts #} -{% include "solana/_vulnerability_patterns.j2" %} {# chain-specific: signer/owner/PDA/CPI checks #} +{% include "rust/vulnerability_patterns_fragment.j2" %} {# shared: overflow, panics, unwrap, lossy casts #} +{% include "solana/vulnerability_patterns_fragment.j2" %} {# chain-specific: signer/owner/PDA/CPI checks #} ``` -`rust/_vulnerability_patterns.j2` (the language facet) states language-level vulnerability +`rust/vulnerability_patterns_fragment.j2` (the language facet) states language-level vulnerability patterns — integer overflow/underflow, `panic!`/`unwrap`/`expect` aborts, lossy conversions, -unchecked results — independent of any chain; `solana/_vulnerability_patterns.j2` adds the +unchecked results — independent of any chain; `solana/vulnerability_patterns_fragment.j2` adds the Solana-native ones. Because the Rust facet is factored out this way, it is reusable by any future Rust chain without copying. --- -## 5. Driver integration +## 5. The Soroban ecosystem (`RUST ⊕ soroban`) + +`SOROBAN` is the `RUST` language facet plus the Soroban/Stellar model and prompts. It is front half +only: there is no Soroban backend yet. Registering it still matters because the prompt templates +are typed, listed in `template_manifest.json`, and rendered by +[tests/test_fuzzed_templates.py](../tests/test_fuzzed_templates.py). + +```python +SOROBAN: SorobanEcosystem = Ecosystem( + name="soroban", + language=RUST, + system_model=SorobanApplication, + analysis_prompts=PromptPair(SOROBAN_ANALYSIS_SYSTEM_TEMPLATE, SOROBAN_ANALYSIS_INITIAL_TEMPLATE), + property_prompts=PropertyPrompts(SOROBAN_PROPERTY_SYSTEM_TEMPLATE, _render_soroban_property_prompt), + validate_analysis=_soroban_validate, + locate_main=_soroban_locate_main, + supports_greenfield=False, + units=_soroban_units, + unit_type=SorobanComponentInstance, + analysis_extra_input=_soroban_analysis_extra_input, + code_explorer_prompt=SOROBAN_CODE_EXPLORER_TEMPLATE, # rust/_common + require_auth / storage kind +) +``` + +- **System model** ([composer/spec/soroban/model.py](../composer/spec/soroban/model.py)) models + contracts, entry-point functions, `Address` auth checks, storage entries, calls, components, and + external actors. +- **Storage** carries its kind everywhere: `instance`, `persistent`, or `temporary`. A component's + `storage_keys` resolve to full `StorageEntry` objects so the templates can show the storage kind. +- **Auth** is explicit. A function with no `require_auth` is represented as `auth == []`; the + templates render that fact instead of hiding it. +- **Units** are one `SorobanComponentInstance` per component of the main contract. +- **Validation** checks duplicate contract identifiers/names, duplicate function slugs, duplicate + component names/slugs, unknown component links, unknown storage keys, duplicate storage keys, and + functions that belong to no component. +- **Templates** include `soroban/platform_model_fragment.j2` in both the analysis and property prompts, so + Soroban execution facts live in one shared place. See + [composer/templates/soroban/README.md](../composer/templates/soroban/README.md). +- **Backend guidance** still belongs to the backend. A future Certora Sunbeam backend should provide + guidance for CVLR `#[rule]` specs and `certoraSorobanProver`; the Soroban ecosystem should only + describe the app and propose properties. + +--- + +## 6. Driver integration `run_pipeline` ([composer/pipeline/core.py](../composer/pipeline/core.py)) takes an `ecosystem` and never hardcodes a domain. It is a required keyword argument — every caller names @@ -230,15 +285,15 @@ async def run_pipeline[..., U, Main, App]( under `BaseApplication` rather than subtypes — so `ecosystem.validate_analysis` does not typecheck there and it names `validate_solidity_connectivity` directly. - **`_extract_all`** iterates `ecosystem.units(main)`, running one property-inference agent per - unit — one per component for EVM, one for the whole program for Solana. + unit — one per component for EVM, Solana, and Soroban. --- -## 6. What is shared and domain-neutral +## 7. What is shared and domain-neutral - Source tools (`fs_tools`, `code_explorer`, `code_document_ref`) — language-neutral; they read - Rust as well as Solidity. The only ecosystem inputs are the `forbidden_read` default and the - explorer prompt string. + Rust as well as Solidity. The language input is the `forbidden_read` default; the explorer + system prompt is an ecosystem template (shared protocol + chain look-fors). - The report (`collect` / `Verdict` / schema) and `ReportBackend`. - Caching, the multi-round property loop, interactive refinement, and the agent plumbing. - The backend seam itself — a verification backend is "just another backend," paired to an @@ -246,7 +301,7 @@ async def run_pipeline[..., U, Main, App]( --- -## 7. Key files +## 8. Key files | Concern | File | |---|---| @@ -257,5 +312,9 @@ async def run_pipeline[..., U, Main, App]( | `FeatureUnit` protocol | [composer/spec/system_model.py](../composer/spec/system_model.py) | | EVM system model + prompts | [composer/spec/system_model.py](../composer/spec/system_model.py) · `composer/templates/application_analysis_*.j2` · `property_analysis_*.j2` | | Solana system model | [composer/spec/solana/model.py](../composer/spec/solana/model.py) | -| Solana prompts + shared Rust fragment | `composer/templates/solana/*.j2` · `composer/templates/rust/_vulnerability_patterns.j2` | +| Solana prompts + shared Rust fragment | `composer/templates/solana/*.j2` · `composer/templates/rust/vulnerability_patterns_fragment.j2` | +| Code-explorer prompts (per ecosystem) | `composer/templates/code_explorer/` · `Ecosystem.code_explorer_prompt` | +| Soroban system model | [composer/spec/soroban/model.py](../composer/spec/soroban/model.py) | +| Soroban prompts (+ platform primer) | `composer/templates/soroban/*.j2` · [its README](../composer/templates/soroban/README.md) | +| Template manifest (what the fuzzer renders) | [template_manifest.json](../template_manifest.json) · [composer/scripts/template_manifest.py](../composer/scripts/template_manifest.py) | | fs-exclusion default (EVM) | [composer/spec/util.py](../composer/spec/util.py) | diff --git a/docs/formalization-abstraction.md b/docs/formalization-abstraction.md new file mode 100644 index 00000000..385c8751 --- /dev/null +++ b/docs/formalization-abstraction.md @@ -0,0 +1,728 @@ +# Design Doc — The Formalization Abstraction + +> Detailed design of how AutoProver turns *extracted properties* into *verified +> artifacts*, the abstraction that makes it backend-agnostic, and a concrete +> walk-through of the CVL (Certora Prover) implementation. +> +> Companion to [ARCHITECTURE.md](../ARCHITECTURE.md). Where that document maps the +> whole system, this one zooms into a single seam: the contract between the generic +> pipeline driver and a verification backend. + +--- + +## 1. Problem & motivation + +The pipeline has two kinds of work: + +- **Shared work** that is identical no matter what you generate — analyze the system + into components, infer a list of properties per component, cache expensive results, + and assemble a final report. +- **Backend-specific work** — *how* a property becomes a checkable artifact, and *how* + that artifact's pass/fail verdict is obtained. For the CVL backend this is "author a + `.spec`, run the Certora Prover, revise on counterexamples." For the Foundry backend + it is "write a `.t.sol`, run `forge test`." + +The **formalization abstraction** is the seam between the two. It lets the driver in +[composer/pipeline/core.py](../composer/pipeline/core.py) own all the shared work while +delegating every backend-specific decision through a small, typed protocol. The CVL and +Foundry backends are two implementations of that protocol; the driver never imports +either. + +### Design goals + +1. **The driver inspects nothing backend-specific.** It moves opaque `FormT` values + around; only the backend ever looks inside them. +2. **No half-initialized state.** Each phase yields an immutable object that is the + constructor input to the next, so ordering is enforced by the type system, not by + call-order discipline. +3. **One result type threads everything.** A single generic parameter `FormT` keys the + cache, the artifact store, the verdict fetcher, and the report — so they cannot drift + out of agreement. +4. **Concurrency is structural.** Pre-formalization setup overlaps property extraction; + per-component formalization fans out — all expressed in the driver, inherited by every + backend for free. + +--- + +## 2. The phase chain + +Formalization is the tail of a three-link immutable chain. Each arrow is a method whose +return type is the input to the next link: + +```text + PipelineBackend ──preflight──▶ Pre ──prepare_system──▶ PreparedSystem ──prepare_formalization──▶ Formalizer + (config, analysis spec, (backend (.main: located main (formalize / verdicts / + artifact store) pre-work) contract; backend setup) report inputs / finalize) +``` + +The driver (`run_pipeline` in [core.py](../composer/pipeline/core.py)) sequences them: + +```python +# 1. analysis-independent backend pre-work runs CONCURRENTLY with the shared analysis +try: + async with asyncio.TaskGroup() as overlap: # the gate: either failure cancels the other + preflight_task = overlap.create_task(backend.preflight(run)) + analysis_task = overlap.create_task(run.runner(TaskInfo(SYSTEM_ANALYSIS_TASK_ID, ...), ...)) +except BaseExceptionGroup as eg: + if len(eg.exceptions) == 1: # grouping is the group's doing, not a contract + raise eg.exceptions[0] from None + raise +preflight, analyzed = preflight_task.result(), analysis_task.result() + +# 2. backend transform: prover lifts to a harnessed app; foundry is identity +prepared = await backend.prepare_system(analyzed, run, preflight) + +# 3. pre-formalization setup runs CONCURRENTLY with property extraction +staged_task = asyncio.create_task(prepared.prepare_formalization(run)) +batches = await _extract_all(prepared.main, ...) +staged = await staged_task # only overlapped; neither side is cancelled + +# 4. a backend whose units share one artifact handed back a StagedFormalizer instead: the +# artifact is authored HERE, once, from every unit's properties (§3.3) +formalizer = await staged.begin(batches, run) if isinstance(staged, StagedFormalizer) else staged + +# 5. per-component formalization (parallel), cache-wrapped by the driver +settled = await asyncio.gather(*[_run(b) for b in batches], return_exceptions=True) +await formalizer.finalize(outcomes, run) + +# 6. shared: build + persist the report from the outcomes + backend verdicts +report = await build_report(..., fetch_verdicts=formalizer.fetch_verdicts) +``` + +The key structural point is the two overlaps, and both fall out of the driver generically. For the +CVL backend, launching `prepare_formalization` before awaiting extraction is what overlaps the slow +AutoSetup / summary / structural-invariant work with per-component property inference — so Foundry +gets the same overlap with zero extra code. + +`preflight` is the earlier peer, for pre-work that needs *nothing at all* from the run: Crucible +builds the program under test and gates a skeleton harness through the real toolchain there, since +neither reads the analyzed model. The two share a task group, so whichever side fails first cancels +the one still running: an agent would otherwise keep spending on a run that can no longer complete, +and a workspace build — the run's slowest non-LLM step — would keep running for a result nothing will +read. That is what makes the preflight a *gate*: a broken workspace stops the run while it has spent +at most one partial analysis agent, instead +of surfacing as unfixable compiler errors in the first authored draft, after the whole extraction +phase ([rust-applications.md §4.2](./rust-applications.md)). + +--- + +## 3. The contract + +Three protocols + three abstract bases define the entire seam. The `PipelineBackend` protocol and +the abstract bases (`Formalizer`, `StagedFormalizer`, `PreparedSystem`) live in +[composer/pipeline/core.py](../composer/pipeline/core.py); the data types the driver moves around +(`BackendResult`, `GaveUp`, `Delivered`, `BackendJob`, `ComponentOutcome`, `CorePhases`, +`PipelineRun`, `SystemAnalysisSpec`, `CorePipelineResult`) live in its sibling +[ptypes.py](../composer/pipeline/ptypes.py), and the persistence protocols in +[composer/spec/types.py](../composer/spec/types.py). + +### 3.1 The result type: `FormT` + +Everything is generic over one type variable, the *backend result*. It is the +intersection of two narrow protocols: + +```python +# composer/pipeline/ptypes.py +class BackendResult(FormalResult, ReportableResult, Protocol): ... +``` + +- **`FormalResult`** ([types.py](../composer/spec/types.py)) — what *persistence* + needs: `property_checks()`, `commentary`, `artifact_text`. +- **`ReportableResult`** ([report/collect.py](../composer/spec/source/report/collect.py)) + — what the *report* needs: `skipped`, `property_checks()`, `output_link`. + +A backend's concrete result (for CVL, `GeneratedCVL`) structurally satisfies both. The +driver only ever holds it as an opaque `FormT`; it never reads a field. + +### 3.2 `Formalizer[FormT]` — the heart of the abstraction + +```python +# composer/pipeline/core.py +@dataclass +class Formalizer[FormT: BackendResult, U: FeatureUnit](ABC): + formalized_type: type[FormT] # the concrete result class — the cache get/put key + backend_tag: ReportBackend # the report vocabulary, stamped into the report + + @abstractmethod + async def formalize(self, label, feat: U, props, ctx, run) -> FormT | GaveUp: ... + + @abstractmethod + async def fetch_verdicts(self, inp: ReportComponentInput[FormT]) -> dict[RuleName, Verdict]: ... + + def extra_report_inputs(self) -> list[ReportComponentInput[FormT]]: + return [] # synthetic report rows; default none + + async def finalize(self, outcomes, run) -> None: + return None # run-level artifacts from the full outcome set; default none +``` + +The contract is deliberately small — one required producer (`formalize`), one required +reader (`fetch_verdicts`), and two optional hooks. The second type parameter `U` is the +*formalized unit* the backend consumes (EVM's `ContractComponentInstance`, a Rust backend's +`FeatureUnit`): the backend reads its concrete unit's members without casts while the driver stays +unit-agnostic. Crucially, a `Formalizer` is **immutable and fully constructed** by whatever produced +it: it carries its prover config, resources, and tool as constructor state, never set post-hoc. By +the time `formalize` runs, every dependency is already present. + +### 3.3 `PreparedSystem` — the formalizer factory, and `StagedFormalizer` + +```python +# composer/pipeline/core.py +@dataclass +class PreparedSystem[FormT: BackendResult, U: FeatureUnit, Main](ABC): + main: Main # the ecosystem's located "main" + @abstractmethod + async def prepare_formalization( + self, run + ) -> Formalizer[FormT, U] | StagedFormalizer[FormT, U]: ... +``` + +`main` is the *ecosystem's* main type (EVM's `ContractInstance`, Solana's +`SolanaProgramInstance`), a different axis from `U` — the driver treats it opaquely and only hands +it to `ecosystem.units(main)`. + +The union return is for a backend whose units all build on **one shared artifact** (Crucible's +fixture; whatever setup module a CVLR backend needs). Such an artifact must be authored from the +union of every unit's properties, which pins it to exactly one point in the run — and +`prepare_formalization` is not it, because that overlaps extraction, so no properties exist there +yet. Nor can it be lazy on first `formalize`: whichever unit won the race would decide the artifact +every other unit is then told to work within — harmless at one unit, silently wrong at several. So +such a backend returns a `StagedFormalizer` instead, and the driver calls its one method between +extraction and the fan-out: + +```python +class StagedFormalizer[FormT: BackendResult, U: FeatureUnit](ABC): + @abstractmethod + async def begin(self, jobs: Sequence[BackendJob[U]], run) -> Formalizer[FormT, U]: ... +``` + +Which of the two a backend returns is its own declared signature, so a backend with no shared +artifact never mentions staging at all. The prover is one of those: its shared peer +(`invariants.spec`) is produced inside `prepare_formalization` (§4.2), because the invariants are +formulated from the *model*, not from the extracted properties. + +### 3.4 The outcome types the driver produces + +```python +# composer/pipeline/ptypes.py +@dataclass(frozen=True) +class Delivered[FormT: BackendResult]: # a success + the path it was written to + result: FormT + deliverable: Path + # unit_file (the deliverable's basename — the verdict-disambiguation key) and + # run_link (the result's output_link) are derived properties. + +class GaveUp(BaseModel): # the single unified give-up signal + reason: str + +@dataclass +class ComponentOutcome[FormT: BackendResult, U: FeatureUnit](BackendJob[U]): + result: Delivered[FormT] | GaveUp | BaseException # success / declined / crashed +``` + +`ComponentOutcome` is a closed sum of the three things that can happen to one component: +it was `Delivered`, the agent `GaveUp` with a reason, or it raised. The driver's `_tally` +folds these into the `CorePipelineResult`; the report phase renders each. + +--- + +## 4. The CVL backend, method by method + +The prover implementation lives in +[composer/spec/source/pipeline.py](../composer/spec/source/pipeline.py). It declares: + +```python +@dataclass +class ProverBackend(PipelineBackend[ + AutoProvePhase, GeneratedCVL, None, SpecIdentity, ContractComponentInstance, + ContractInstance, SourceApplication, None, +]): + backend_guidance = CERTORA_BACKEND_GUIDANCE + core_phases = CorePhases({"analysis": ..., "extraction": ..., + "formalization": AutoProvePhase.CVL_GEN, "report": AutoProvePhase.REPORT}) + analysis_spec = SystemAnalysisSpec(COMMON_SYSTEM_CACHE_KEY, AP_PROPERTIES_KEY_NAME) + + _store: ProverArtifactStore + _prover_opts: ProverOptions + + @property + @override + def artifact_store(self) -> ProverArtifactStore: return self._store +``` + +So `FormT = GeneratedCVL`, the artifact id type `A = SpecIdentity`, and the phase enum is +`AutoProvePhase`. The seam is structural, so naming `PipelineBackend` as a base is optional — but +that is the one place those eight arguments can be written down and tied to each other, and it +moves the conformance check from wherever the backend reaches `run_pipeline` to where it is +*defined*. + +Three of the four non-method members are run-constants the driver only reads, so they are stated as +plain class attributes. The store is the exception, and is declared **read-only** on the protocol so +this backend can narrow it: the driver needs only `ArtifactStore[SpecIdentity, GeneratedCVL]`, while +`ProverPrepared` needs the `ProverArtifactStore` this returns (for `write_component_runs`), and a +mutable attribute is invariant — a narrowed field would not typecheck. A backend with nothing to +narrow returns its store the same way; `RustBackend` derives its guidance and analysis spec from the +wheel's descriptor in `__post_init__` rather than by accessor, for the same reason. + +### 4.1 `prepare_system` — harness lift + +```python +async def prepare_system( + self, analyzed: SourceApplication, run, preflight: None, +) -> PreparedSystem[GeneratedCVL, ContractComponentInstance, ContractInstance]: + sys_desc = await run.runner(TaskInfo(HARNESS_TASK_ID, ...), lambda: run_harness_creation(...)) + harnessed = _lift_harnessed(analyzed, sys_desc) # SourceApplication → HarnessedApplication + prover_tool = get_prover_tool(run.env.llm_heavy(), run.source.contract_name, + run.source.project_root, prover_opts=self._prover_opts) + return ProverPrepared(main_instance(harnessed, run.source), self._store, + sys_desc, harnessed, prover_tool, self._prover_opts, analyzed) +``` + +It runs the harness-classification agent, folds the generated harnesses back into the app +as a `HarnessedApplication` (`_lift_harnessed` in [pipeline.py](../composer/spec/source/pipeline.py)), builds +the shared `verify_spec` prover tool once, and packages everything the next phase needs into +an immutable `ProverPrepared`. (Foundry's `prepare_system` is an identity transform — no +harness, no tool.) + +### 4.2 `prepare_formalization` — the concurrent setup fan-out + +This is where the CVL backend does its expensive pre-work, and it is the richest method in +the abstraction. From `ProverPrepared` in [pipeline.py](../composer/spec/source/pipeline.py): + +```python +async def prepare_formalization(self, run) -> Formalizer[GeneratedCVL, ContractComponentInstance]: + # AutoSetup (+ custom summaries) ∥ structural-invariant formulation — both depend only + # on the harnessed app, so they run concurrently. + (setup_config, resources), invariants = await asyncio.gather( + self._autosetup(run), self._invariants(run), + ) + + invariant = None + if invariants.inv: + inv_props = [PropertyFormulation(title=inv.name, description=inv.description, sort="invariant") + for inv in invariants.inv] + self._store.write_properties(InvariantSpec(), inv_props) + + # Generate invariants.spec ONCE, with cache short-circuit + inv_cvl_ctx = run.ctx.child(INV_CVL_KEY) + cached = await inv_cvl_ctx.cache_get(GeneratedCVL) + if cached is not None: + inv_cvl = cached + else: + inv_result = await run.runner(TaskInfo(INVARIANT_CVL_TASK_ID, ...), + lambda: batch_cvl_generation(inv_cvl_ctx.abstract(CVLGeneration), + setup_config.prover_config, inv_props, None, resources, self._prover_tool, ...)) + if isinstance(inv_result, GaveUp): + raise RuntimeError(f"Structural invariant CVL generation gave up: {inv_result.reason}") + inv_cvl = inv_result + await inv_cvl_ctx.cache_put(inv_cvl) + + inv_path = self._store.write_artifact(InvariantSpec(), inv_cvl) + # Append invariants.spec to the resource set so EVERY per-component spec imports it. + resources = [*resources, CVLResource(path=inv_path, required=False, + description="Structural invariants that may be assumed as preconditions", sort="import")] + invariant = (inv_props, Delivered(inv_cvl, inv_path)) + + return ProverRunner(GeneratedCVL, "prover", self._store, self._prover_tool, + setup_config.prover_config, resources, invariant, make_prover_fetcher()) +``` + +Three things worth calling out: + +- **Concurrency inside the method.** AutoSetup+summaries and invariant *formulation* are + independent, so they `gather`. This nests under the driver-level overlap (this whole + method already runs concurrently with property extraction). +- **Structural invariants are formalized eagerly, here, not per-component.** They are + generated once into `invariants.spec`, then injected into `resources` so every later + per-component spec can `import` them as preconditions. The invariant CVL goes through the + exact same `batch_cvl_generation` path that components do (with `component=None`). +- **The returned `ProverRunner` is fully loaded.** Its config, resource set (now including + `invariants.spec`), prover tool, and the in-memory invariant result are all constructor + fields. `formalize` adds nothing — it only *reads* them. + +### 4.3 `formalize` — per-component authoring + verification loop + +This is one instance of the **authoring session** (§4.3.1), which every backend runs. + +The `Formalizer.formalize` impl is a thin adapter; all the work is in +`batch_cvl_generation`: + +```python +# ProverRunner.formalize (pipeline.py) +async def formalize(self, label, feat, props, ctx, run) -> GeneratedCVL | GaveUp: + return await batch_cvl_generation( + ctx.abstract(CVLGeneration), self._prover_config, props, feat, + self._resources, self._prover_tool, run.env, label, run.source, SPECS_DIR) +``` + +`batch_cvl_generation` ([author.py](../composer/spec/source/author.py)) builds a +dedicated LLM agent graph and runs it to a fixpoint. The agent is given: + +- the property batch + component context, rendered into the prompt; +- the resource set as `import` views, with paths made relative to the spec dir so the + prover resolves CVL `import`s correctly; +- a tool belt: CVL authoring tools, the `verify_spec` prover tool, config-edit tools, and the + completion/give-up/expectation tools (`PublishResultTool`, `GiveUpTool`, + `ExpectRuleFailure`/`ExpectRulePassage`). + +Two **hard validation gates** must both pass before the agent may publish +([author.py](../composer/spec/source/author.py)): + +```python +required_validations=[FEEDBACK_VALIDATION_KEY, PROVER_VALIDATION_KEY] +``` + +- the **prover gate** — the spec must actually run; +- the **feedback gate** — a separate `property_feedback_judge` agent + ([composer/spec/feedback.py](../composer/spec/feedback.py)) adjudicates whether each + property is genuinely covered, and the author may file evidence-backed `Rebuttal`s + (typecheck failure / counterexample / manual citation / reasoned) against prior feedback. + +The agent loop ends in exactly one of two states, mapped onto the abstraction's +`FormT | GaveUp`: + +```python +if res_state["failed"]: + return GaveUp(reason=res_state["result"]) +return GeneratedCVL(commentary=..., cvl=..., skipped=..., property_rules=..., + config=res_state["config"], final_link=res_state.get("prover_link")) +``` + +Note `config` and `final_link` are captured into the result. That is deliberate: a later +cache hit skips the prover entirely, so the result must carry enough to rebuild +`certora/confs/` and keep the run link without re-running anything. + +### 4.3.1 The authoring session, shared by every backend + +Every backend authors the same way, and that shape lives in +[composer/authoring/](../composer/authoring/). It is **one stateful agent**, not a retry loop +around a stateless one — the difference is where the state lives, and everything else follows from +it: + +- **One `curr_spec` buffer.** `put_spec` replaces it; an edit tool replaces one exact span + ([composer/core/edit.py](../composer/core/edit.py)) and is what the agent should reach for once a + draft exists. A backend that can reject a malformed spec cheaply supplies a validator, and a + rejected write leaves the buffer untouched. +- **Gates stamp, they do not return.** A checker or judge that accepts the draft writes + `spec_digest(buffer, skips)` into `validations`. `check_completion` then requires every + `required_validations` key to carry a stamp equal to the digest *as it now stands*, so editing + after a green run invalidates that run without anything having to remember to clear it. The digest + covers the skip declarations too, because "this property is left out, here is why" is part of what + was accepted. +- **A judge that must state a verdict.** `build_feedback_judge` compiles a sub-agent with the + session's tool belt, a rough-draft scratchpad, the run memory, and an enforced read-back of the + draft (`did_read` — reviewing the copy in its own prompt is not reviewing what was written). It + returns a structured `PropertyFeedback`, so there is no unparseable reply to interpret. The author + may answer a prior round with an evidence-typed `Rebuttal` rather than re-arguing it. +- **Two honest exits.** `record_skip` excuses a property from the publish-time mapping with a + justification; `give_up` ends the session with a reason that reaches the report. Both are better + outcomes than a spec that only looks checked. +- **A publish gate that checks the mapping.** `validate_check_mapping` requires every non-skipped + property to name at least one check, refuses a skipped one, and — when the backend's checker + reports what it ran (forge names every test; the prover names nothing) — checks both directions + against that ground truth. + +What a backend supplies is the part that genuinely differs: the put-time validator, the gate tools +and the keys they stamp, the mapping's ground truth, the prompts, and its **own vocabulary**. That +last one is deliberate: `MappingVocab` carries the word each backend uses with its own model — CVL +says *rule*, Foundry says *test*, a Rust wheel declares its own via `check_noun` — because an author +writes better in the language its generated code already uses. The framework's generic term for the +concept is a **check**: the backend's named, runnable verification of one property, which yields a +`Verdict`. + +One holdout, deliberate: the **report** still says *rule* (`RuleName`, `total_rules`, +`FormalizedProperty.rules`). Those are field names in `certora/ap_report/report.json`, which the +standalone renderer reads back, so renaming them changes an output format rather than an internal +name. `type RuleName = CheckName` marks the seam, and `fetch_verdicts` is the one place the two +vocabularies meet. + +The per-backend assembly (which tools, which prompts, which cache) stays in that backend's own entry +point — `batch_cvl_generation`, `batch_foundry_test_generation`, +[`run_session`](../composer/rustapp/session.py). They differ in exactly the parameters the core takes, +so collapsing them into one function would buy nothing. + +### 4.4 `extra_report_inputs` — folding in the invariants + +Per-component outcomes are assembled by the driver. The structural invariants are a +*synthetic* component the backend contributes ([pipeline.py](../composer/spec/source/pipeline.py)): + +```python +def extra_report_inputs(self) -> list[ReportComponentInput[GeneratedCVL]]: + if self._invariant is None: + return [] + inv_props, inv = self._invariant + return [ReportComponentInput(name="Structural Invariants", props=inv_props, formalized=inv)] +``` + +This is the report-side payoff of formalizing invariants in `prepare_formalization`: the +in-memory `Delivered[GeneratedCVL]` is replayed straight into the report with no special +casing in the driver. + +### 4.5 `fetch_verdicts` — pass/fail per rule + +```python +async def fetch_verdicts(self, inp) -> dict[RuleName, Verdict]: + return await self._fetch(inp) # make_prover_fetcher(): queries ProverOutputUtility off-thread +``` + +The fetcher resolves each spec's prover run (via `inp.formalized.run_link`) and rolls per-rule +outcomes into `Verdict`s. The `collect` step +([report/collect.py](../composer/spec/source/report/collect.py)) then keys rules by +`(unit_file, name)` so a structural invariant imported into several component specs collapses +to one entry, and uses `Verdict.merge` (priority `BAD > ERROR > TIMEOUT > UNKNOWN > GOOD`) to +roll up multiple results for one rule. Foundry's fetcher instead reads pass/fail straight off +the result with no run service — same protocol, different source. + +### 4.6 `finalize` — run-level artifact + +```python +async def finalize(self, outcomes, run) -> None: + runs = {ComponentSpec(o.feat.slugified_name).run_key: o.result.run_link + for o in outcomes if isinstance(o.result, Delivered) and o.result.run_link} + if self._invariant and self._invariant[1].run_link: + runs[InvariantSpec().run_key] = self._invariant[1].run_link + self._store.write_component_runs(runs) # → components_to_prover_runs.json +``` + +`finalize` is the one hook that sees the *entire* outcome set at once — used here to emit the +`{spec → prover-run link}` map. Foundry omits it (default no-op). + +--- + +## 5. The result type as the central key + +`GeneratedCVL` ([cvl_generation.py:125](../composer/spec/cvl_generation.py)) is the concrete +`FormT`. It satisfies `BackendResult` structurally — note nothing declares +`class GeneratedCVL(BackendResult)`; the protocols match by shape: + +```python +class GeneratedCVL(BaseModel): + commentary: str + cvl: str + skipped: list[SkippedProperty] = Field(default_factory=list) + property_rules: list[PropertyRuleMapping] = Field(default_factory=list) + config: dict | None = None + final_link: str | None = None + + def property_checks(self) -> list[tuple[str, list[str]]]: # FormalResult + ReportableResult + return [(m.property_title, m.rules) for m in self.property_rules] + + @property + def artifact_text(self) -> str: # FormalResult: bytes to write + return self.cvl + + @property + def output_link(self) -> str | None: # ReportableResult: run link + return _output_link(self.final_link) # /jobStatus/ → /output/ +``` + +The same value is the key for four otherwise-independent subsystems, which is what keeps them +from disagreeing: + +| Consumer | Uses | Via | +|---|---|---| +| **Cache** | the *type* `GeneratedCVL` | `formalizer.formalized_type` → `cache_get`/`cache_put` | +| **Artifact store** | `artifact_text`, `commentary`, `property_checks()` | `ArtifactStore.write_artifact` | +| **Report** | `skipped`, `property_checks()`, `output_link` | `ReportableResult` | +| **Run-link map** | the persisted `final_link` | `finalize` | + +--- + +## 6. Persistence: the artifact store + +`Delivered` pairs a result with the path it was written to — and those two always travel +together because the path *exists only because the result did* +([ptypes.py](../composer/pipeline/ptypes.py)). The write happens in the driver's `_run` +closure: + +```python +backend.artifact_store.write_properties(result_key, batch.props) # before generation +... +Delivered(result, backend.artifact_store.write_artifact(result_key, result)) # after success +``` + +The store is generic over `(ArtifactIdentifier, FormalResult)` +([artifacts.py](../composer/spec/artifacts.py)). The base writes everything that is +identical across backends — `properties.json`, `commentary.md`, the +`{property title → the checks demonstrating it}` map — keyed off the identifier's `stem`. The CVL +subclass [ProverArtifactStore](../composer/spec/source/artifacts.py) adds the +CVL-specific bundle: it overrides `write_artifact` to also emit a `.conf` (base config + +fixed run overlay) alongside the `.spec`. + +The artifact id is itself a small sum type, so naming conventions live in one place rather +than being interpolated at call sites: + +```python +@dataclass(frozen=True) +class ComponentSpec: # autospec_.spec + slug: str + @property + def stem(self): return f"autospec_{self.slug}" + @property + def run_key(self): return self.slug + +@dataclass(frozen=True) +class InvariantSpec: # invariants.spec + @property + def stem(self): return "invariants" +``` + +`ProverBackend.to_artifact_id(component)` maps a component instance to its `ComponentSpec`; +the driver uses it both to write properties before generation and to write the artifact after. + +The resulting on-disk layout (all under the project's `certora/`): + +``` +certora/specs/autospec_.spec # per-component CVL +certora/specs/invariants.spec # structural invariants (imported by the above) +certora/confs/.conf # prover config per spec +certora/properties/.properties.json # inferred properties +certora/properties/.property_rules.json # property → [rule names] +certora/properties/.commentary.md # author's commentary +certora/ap_report/report.json # final cross-referenced report +.certora_internal/autoProve/components_to_prover_runs.json # finalize() output +``` + +--- + +## 7. Caching wraps formalization (driver-owned) + +A backend never writes cache logic — the driver does, keyed by +`formalizer.formalized_type` (the `_run` closure in [core.py](../composer/pipeline/core.py)): + +```python +async def _run(batch): + result_key = backend.to_artifact_id(batch.feat) + backend.artifact_store.write_properties(result_key, batch.props) + child = await batch.feat_ctx.child(_batch_cache_key(batch.props), {...}) + cached = await child.cache_get(formalizer.formalized_type) # ← type comes from the formalizer + if cached is None: + result = await run.runner(TaskInfo(formalize_task_id(...)), + lambda: formalizer.formalize(label, batch.feat, batch.props, child, run)) + if not isinstance(result, GaveUp): + await child.cache_put(result) + else: + result = cached + ... +``` + +The cache key is the hash of the *property batch* (`_batch_cache_key`), under the component's +context, under the `properties` context — the hierarchical scheme described in +[ARCHITECTURE.md §7](../ARCHITECTURE.md). Because the result type carries `config` and +`final_link`, a cache hit can rebuild the `.conf` and keep the run link without touching the +prover. The structural-invariant CVL has its own cache short-circuit inside +`prepare_formalization` (`INV_CVL_KEY`, §4.2) for the same reason. + +--- + +## 8. Failure handling + +The abstraction encodes three distinct failure modes, each handled differently: + +| Failure | Representation | Driver behavior | +|---|---|---| +| Agent declines a component | `formalize` returns `GaveUp(reason)` | recorded as a `ComponentOutcome`, surfaced in `failures`, rendered in report as a gap; **not cached** | +| Component crashes | `formalize` raises | `asyncio.gather(..., return_exceptions=True)` captures it into `ComponentOutcome.result` | +| Invariant CVL gives up | `prepare_formalization` raises `RuntimeError` | **fatal** — invariants are a shared precondition, so the whole run aborts | +| Report build fails | exception in `build_report` | best-effort: logged, run still succeeds — unless `report_build.RERAISE_REPORT_FAILURES` is set, which tests flip to make a silent report failure fail loudly | + +The asymmetry is intentional: a single component giving up is a normal, reportable outcome, +but the shared invariant spec failing would silently weaken every downstream component, so it +fails loud. + +--- + +## 9. Extending: what a new backend must provide + +To add a backend you implement `PipelineBackend` and the three phase objects — nothing in the +driver changes. The Foundry backend +([composer/foundry/pipeline.py](../composer/foundry/pipeline.py)) is the proof: it reuses +system analysis, property extraction, caching, and the report, and contributes only: + +| Abstraction member | CVL backend | Foundry backend | +|---|---|---| +| `FormT` | `GeneratedCVL` | `GeneratedFoundryTest` | +| `preflight` | none (its pre-work needs the harnessed model) | none (`forge` builds the project already) | +| `prepare_system` | harness lift + prover tool | identity | +| `prepare_formalization` | AutoSetup ∥ summaries ∥ invariants | trivial (pre-built formalizer) | +| shared artifact (`StagedFormalizer`) | none — `invariants.spec` is built from the model, in `prepare_formalization` | none | +| `formalize` | authoring session, gated by `verify_spec` | authoring session, gated by `forge_test` | +| `fetch_verdicts` | query prover output off-thread | read ran/expected tests off the result | +| `extra_report_inputs` | synthetic "Structural Invariants" | none | +| `finalize` | `components_to_prover_runs.json` | none | +| artifact bundle | `.spec` + `.conf` | `.t.sol` + metadata | + +A backend author's checklist: + +1. Define a result type satisfying `FormalResult` + `ReportableResult` (`artifact_text`, + `commentary`, `property_checks()`, `skipped`, `output_link`). +2. Subclass `ArtifactStore` for the on-disk bundle; define an `ArtifactIdentifier` sum type. +3. Implement `PipelineBackend` (`preflight`, `prepare_system`, `to_artifact_id`, + `backend_guidance`, `core_phases`, `analysis_spec`, `artifact_store`) — naming it as a base and + filling in its eight type arguments is what the in-tree backends do. `preflight` returns `None` + unless the backend has pre-work that needs nothing from the run — if it must *build* something, + that is where the build belongs, so it overlaps system analysis and can fail the run before the + model has been spent (Crucible: [rust-applications.md §4.2](./rust-applications.md)). +4. Implement `PreparedSystem.prepare_formalization` returning a fully-constructed + `Formalizer` — or, if every unit builds on one shared artifact, a `StagedFormalizer` whose + `begin` authors it from the union of all units' properties (§3.3). +5. Implement `Formalizer.formalize` + `fetch_verdicts`; override `extra_report_inputs` / + `finalize` only if needed. `formalize` should assemble the shared authoring session (§4.3.1) + rather than grow its own loop — what it supplies is the gate tools, the prompts, and its own + noun for a check. + +--- + +## 10. End-to-end trace (CVL backend) + +Putting it together, one run of the CVL backend over a component: + +``` +_entry_point → cli_pipeline → cont(env, ProverBackend, EVM) +└─ run_pipeline(ProverBackend, run, ecosystem=EVM) + 1. ┌ create_task: ProverBackend.preflight ─▶ None # a building backend puts its build here + └ run_component_analysis ─────────────▶ SourceApplication # concurrent with the above + 2. ProverBackend.prepare_system + run_harness_creation ─▶ SystemDescriptionHarnessed + _lift_harnessed ─▶ HarnessedApplication + get_prover_tool ─▶ verify_spec tool + ▶ ProverPrepared(main=located main contract, ...) + 3. ┌ create_task: ProverPrepared.prepare_formalization + │ gather( _autosetup → (config, [summaries]) , _invariants → [BaseInvariant] ) + │ batch_cvl_generation(component=None) ─▶ invariants.spec (cached under INV_CVL_KEY) + │ resources += invariants.spec + │ ▶ ProverRunner(config, resources, invariant, fetch) + └ _extract_all ─▶ [ _Batch(component, props) , ... ] # runs concurrently with the above + 4. for each batch (parallel, semaphore-bounded): + write_properties(ComponentSpec(slug), props) + cache_get(GeneratedCVL)? ── hit ─▶ reuse + └ miss ─▶ ProverRunner.formalize + batch_cvl_generation(component=feat) + author CVL ⇄ verify_spec ⇄ feedback judge (loop) + gates: PROVER_VALIDATION + FEEDBACK_VALIDATION + ─▶ GeneratedCVL | GaveUp + cache_put(result) + write_artifact ─▶ autospec_.spec (+ .conf) ⇒ Delivered(result, path) + ─▶ ComponentOutcome + ProverRunner.finalize(outcomes) ─▶ components_to_prover_runs.json + 5. build_report( per-component inputs + extra_report_inputs(), + fetch_verdicts=ProverRunner.fetch_verdicts ) ─▶ certora/ap_report/report.json +``` + +--- + +## 11. Key files + +| Concern | File | +|---|---| +| Driver + the seam (`PipelineBackend`, `Formalizer`, `StagedFormalizer`, `PreparedSystem`) | [composer/pipeline/core.py](../composer/pipeline/core.py) | +| The driver's data types (`BackendResult`, `Delivered`, `GaveUp`, `ComponentOutcome`, `PipelineRun`, …) | [composer/pipeline/ptypes.py](../composer/pipeline/ptypes.py) | +| Result protocols (`FormalResult`, `ArtifactIdentifier`) | [composer/spec/types.py](../composer/spec/types.py) | +| `ReportableResult`, `Verdict`, `VerdictFetcher`, `collect` | [composer/spec/source/report/collect.py](../composer/spec/source/report/collect.py) | +| CVL backend (the three phase objects) | [composer/spec/source/pipeline.py](../composer/spec/source/pipeline.py) | +| The shared authoring session (buffer, stamps, judge, publish gate) | [composer/authoring/](../composer/authoring/) | +| CVL authoring agent (`batch_cvl_generation`) | [composer/spec/source/author.py](../composer/spec/source/author.py) | +| CVL result type (`GeneratedCVL`) | [composer/spec/cvl_generation.py](../composer/spec/cvl_generation.py) | +| Artifact store base / CVL subclass | [composer/spec/artifacts.py](../composer/spec/artifacts.py) · [composer/spec/source/artifacts.py](../composer/spec/source/artifacts.py) | +| Foundry backend (contrast) | [composer/foundry/pipeline.py](../composer/foundry/pipeline.py) | +| A Rust-wheel backend on this seam | [docs/rust-applications.md](./rust-applications.md) | diff --git a/docs/rag-import-format.md b/docs/rag-import-format.md new file mode 100644 index 00000000..52319ed8 --- /dev/null +++ b/docs/rag-import-format.md @@ -0,0 +1,343 @@ +# Design Doc — A common JSON format for RAG entries + a single importer + +> Today every documentation corpus that feeds a search tool ships its **own** RAG builder: +> a bespoke Python module that parses that corpus's native format *and* talks to the RAG +> database, plus a shell wrapper. Adding a new corpus (notably: a new Rust application that +> wants its own `*_kb`) means writing another builder wired to the DB. +> +> This proposes splitting that seam: a **producer** emits a corpus as a common JSON document, +> and one shared **importer** ingests any such document into the RAG DB. The DB coupling, +> chunking, embedding, batching and dual-path ingestion move into the importer — once — and a +> producer shrinks to "parse my docs → emit JSON", with no dependency on the RAG stack. +> +> **Scope:** this layer ships the *mechanism* — the format, the importer, and the tag→connection +> registry — with **no corpus on it**; the first adopter is the Crucible application, which lands +> with the Solana backend. The Foundry and CVL builders stay exactly as they are; they are +> candidate future adopters (§5), not part of this change. The format is nonetheless designed to be +> general, so migrating them later needs no schema change. +> +> Companion to [rust-applications.md](./rust-applications.md) (the descriptor-driven app model and +> the wheel FFI surface). The `knowledge_base` +> tag defined here is the *same* tag a wheel already declares as +> [`rag_db_default`](../composer/rustapp/descriptor.py). + +--- + +## 1. What's actually shared today — and what isn't + +Three builders existed, one per corpus: + +| Builder | Source format | Ingests | +| --- | --- | --- | +| [`ragbuild.py`](../composer/scripts/ragbuild.py) | CVL-manual HTML (docutils) | vector + manual | +| [`foundry_ragbuild.py`](../composer/scripts/foundry_ragbuild.py) | Foundry cheatcode HTML fragments | vector only | +| `crucible_ragbuild.py` (never landed — replaced by this mechanism) | Crucible markdown | vector + manual | + +Each had a shell wrapper ([`populate_rag.sh`](../scripts/populate_rag.sh), +[`populate_foundry_rag.sh`](../scripts/populate_foundry_rag.sh), `populate_crucible_rag.sh`), and +each pins a default connection constant (`DEFAULT_CONNECTION`, `FOUNDRY_DEFAULT_CONNECTION`, +`SANITY_DEFAULT_CONNECTION`) in [`composer/rag/db.py`](../composer/rag/db.py). (This section +describes the state that *motivated* the change: the Crucible builder + wrapper were written that +way first, and this mechanism is what replaced them — the CVL and Foundry ones remain.) + +The important observation: **only the first column differs.** Everything downstream is already +common code the three builders call into: + +- the chunk model [`BlockChunk`](../composer/rag/types.py) — a header path (`h1..h6`), a `part` + index, `code_refs`, and a `chunk` body with `` placeholders; +- the length-bounded splitter `BlockBuilder` / `BuilderConfig` + ([`text_processors.py`](../composer/scripts/text_processors.py)), driven by spaCy; +- the **dual-path ingestion** on [`ComposerRAGDB`](../composer/rag/db.py): + `add_chunks_batch` (embedded chunks → vector search) **and** `add_manual_section` (the full + section → keyword search / `get_section`); +- embedding, batching, and `part`-numbering of repeated header paths. + +So each builder re-implements *source parsing* and then hand-rolls the same orchestration around +the same shared primitives. The bespoke part is small; the boilerplate around it is duplicated +per corpus and, critically, **carries a hard dependency on the RAG DB and the heavy `ragbuild` +uv group** (spaCy + sentence-transformers). A new Rust app that just wants to contribute a corpus +inherits all of that — which is what makes Crucible the natural first case to lift onto a generic +mechanism (the other two builders already exist and work, so they can migrate later or never). + +### The right cut: two declared products, chunked by the importer + +The seam should fall **between parsing and chunking**, not after chunking. But the two indexes +store **different products**, not one product rendered twice: + +- the **manual index** (`manual_sections`) stores *documents* — complete reference units, + addressable by header path, returned whole by `get_section`, at whatever size they are; +- the **vector index** (`documents`) stores *passages* — length-bounded chunks whose cut points + matter, and whose quality depends on knowing what each piece of content *is*: a paragraph that + may be split at sentence boundaries, a table that must stay intact, prose that merely continues + around a code sample. + +The CVL builder keeps two pipelines for exactly this reason: its manual documents are +subtree-inclusive renderings (a section's document contains its child sections inline, with +editorial rules like Example subsections never getting their own document), while its vector +chunks are disjoint bounded passages driven with per-element chunking flags (§5). A single +flattened "section" list feeding both indexes can express neither side faithfully — it erases the +chunking hints the vector side needs, and it forces the manual unit to coincide with the vector +grouping unit. + +So the manifest declares the two products **explicitly and independently** (§2): manual sections +as whole documents, embedded groups as kind-annotated block runs. Laying out both is where the +genuinely corpus-specific editorial judgment lives — often two views of the same source, but +never derived one from the other. The importer owns everything mechanical from there down: +running `BlockBuilder` to cut embedded groups into length-bounded chunks, assembling manual +documents, assigning `` tags, numbering `part`s, embedding, and batching. + +Why this cut and not "emit finished `BlockChunk`s": + +- **Chunking is common, tuned, and heavy.** Length-bounding needs spaCy and a shared + `max_length`. Keeping it in the importer means producers need neither spaCy nor + sentence-transformers — a Rust app can emit the JSON from a trivial script (or from the wheel + itself; see §6) with no RAG dependencies. +- **Code-ref tagging is a footgun.** `crucible_ragbuild` manually tracked a `code_refs` list and + emits `` tags in lockstep; getting that wrong orphans a ref. Producers should + never see the tag scheme — they just say "this block is code." +- **`part` numbering is global.** The manual-section table is unique on + `(h1..h6, part)`; repeated header paths must bump `part`. That's a whole-corpus concern the + importer is positioned to own; a producer emitting isolated chunks can't. + +--- + +## 2. The JSON format + +A corpus is one **manifest** document: metadata plus the two retrieval products. Schema (v1), +mirrored by a pydantic model the way +[`descriptor.py`](../composer/rustapp/descriptor.py) mirrors the Rust `AppDescriptor`: + +```jsonc +{ + "version": 1, + "knowledge_base": "crucible_kb", // logical KB tag (== descriptor rag_db_default) + "source": "crucible@a1b2c3d docs/*.md", // free-text provenance, for logs only + "manual_sections": [ + { + "headers": ["Writing Fuzz Harnesses", "PDA Seed Encoding"], + "blocks": [ + { "kind": "text", "body": "Seeds are encoded as ..." }, + { "kind": "code", "body": "let (pda, bump) = Pubkey::find_program_address(...);" }, + { "kind": "text", "body": "The bump is then ..." } + ] + } + ], + "embedded_groups": [ + { + "headers": ["Writing Fuzz Harnesses", "PDA Seed Encoding"], + "blocks": [ + { "kind": "paragraph", "body": "Seeds are encoded as ..." }, + { "kind": "code", "body": "let (pda, bump) = Pubkey::find_program_address(...);" }, + { "kind": "continuation", "body": "— the bump is then verified by the runtime." }, + { "kind": "atomic", "body": "| seed | meaning |\n| --- | --- |\n| ... | ... |" } + ] + } + ] +} +``` + +Field notes: + +- **`version`** — schema version; the importer refuses any value it doesn't recognize (an exact + match against `SCHEMA_VERSION`), before any DB write. Lets the format evolve without silently + mis-ingesting old files. +- **`knowledge_base`** — the logical corpus tag. This is the *same* string a wheel declares as + `rag_db_default` and that [`rag_env.py`](../composer/tools/rag_env.py) resolves to search + tools. Making producer, importer, and runtime agree on one tag is a real simplification: it + becomes the single key naming a corpus end to end. The importer resolves it to a connection + string via a registry (§4), overridable by `--output`. +- **`source`** — provenance for logging/traceability only. The RAG schema is header-only + ([`documents`](../composer/rag/db.py) / `manual_sections` store `content + h1..h6`), so this + is **not** persisted per row; it just lands in the importer's log line. (If we later want + per-row provenance we'd extend the DB schema — out of scope for v1.) +- **`headers`** — a header path (on both products), at most 6 entries: `_normalize_head` maps + entry *i* to column `h(i+1)`, leaving a falsy or absent level as `NULL` in its own column (so a + gap stays a gap — nothing is left-packed). A path longer than 6 is a producer bug, not something + the importer trims: it raises rather than silently dropping the deepest level. +- **`manual_sections[].blocks`** — ordered `{ "kind": "text" | "code", "body": "..." }`. A manual + section is never split, so the only distinction that matters is prose vs. code: code is held + aside as `code_refs` behind an importer-assigned `` placeholder, keeping it out of + the keyword-searchable text while `get_section` substitutes it back. +- **`embedded_groups[].blocks`** — ordered `{ "kind": ..., "body": "..." }` where the kind + carries the chunking semantics, mapping 1:1 onto the ways the CVL builder already drives + `BlockBuilder`: + + | kind | what the block is | when a chunk overflows | + | --- | --- | --- | + | `paragraph` | a self-contained prose unit | split at sentence boundaries; cuts prefer the block edge, carrying the previous chunk's last sentence as overlap context | + | `atomic` | structure that must survive intact (tables, lists) | never sentence-split — emitted as one oversized chunk | + | `continuation` | prose resuming the stream an earlier block interrupted | no boundary preference; may be cut at any sentence end | + | `code` | a code sample | never split, never embedded as prose — held as `code_refs` behind a `` placeholder | + +**Each product feeds exactly its own index.** The two indexes answer different questions — "what +passage *means* this?" vs. "which documents *contain* this term, and give me one in full" — and a +corpus declares what it wants in each. Typically the two lists are parallel views of the same +source, since a corpus's tools module (`composer/tools/_rag.py`, bound by +[`rag_env.py`](../composer/tools/rag_env.py)) usually binds all three retrieval styles — but +nothing requires it: a vector-only corpus (what Foundry's builder produces today, §5) is an +`embedded_groups`-only manifest, and manual documents may overlap in content (as CVL's +subtree-inclusive documents do) without affecting the vector side at all. + +Deliberately **not** in the schema: `part` (importer-assigned), `code_refs` / `` +tags (importer-assigned), `max_length` / chunking knobs (importer flags — a cross-corpus tuning +concern, not corpus data), and any ingest-path selector — what a corpus ingests is exactly what +it declares. + +--- + +## 3. The importer + +One module, `composer/scripts/rag_import.py`, that factored the shared orchestration out of the +former `crucible_ragbuild`'s `_async_main` (which was already 90% of this) and generalized it over +the manifest — minus the markdown parser, which stays in the producer: + +``` +uv run --isolated --group ragbuild python -m composer.scripts.rag_import \ + corpus.rag.json [more.rag.json ...] [--output ] [--max-length N] [--print] +``` + +Behaviour: + +1. **Load + validate** each manifest against the pydantic model (clear errors on a malformed + file, before any DB write). +2. **Resolve the target** once per manifest: `--output` if given, else the connection registered + for `knowledge_base` (§4). Refuse to run if neither resolves. +3. Ingest each product into its own index: + - **vector** → run a `BlockBuilder` over each embedded group, cutting as each block's kind + dictates (§2); buffer the resulting `BlockChunk`s, flush via `add_chunks_batch` at + `_BATCH_SIZE`; + - **manual** → assemble each manual section into one whole-document `BlockChunk` (code as + `` tags), assign its `part` from a per-header-path counter, + `add_manual_section`. +4. **`--print`** — dry-run: render manual sections and name embedded groups on stdout, no DB + writes (parity with every builder's existing `--print`). + +That is the orchestration the former `crucible_ragbuild` hand-rolled, now reusable by any producer +that emits the manifest. Note it needs the `ragbuild` uv group (spaCy + sentence-transformers) — but +now *only the importer* does; producers don't. + +--- + +## 4. Connection resolution + +The importer resolves a manifest's `knowledge_base` tag to a DB connection via a small registry, +overridable by `--output`: + +```python +KNOWLEDGE_BASES: dict[str, str] = { + # "_kb": CORPUS_DEFAULT_CONNECTION, +} +``` + +It is **empty here** — a corpus's entry lands with the application that declares it, together with +the `composer/tools/_rag.py` that searches it, because [`rag_env.py`](../composer/tools/rag_env.py) +requires both halves before a tag is usable. This is the same registry idea +as `rag_env.py` (tag → search tools) and the ecosystem registry: +a declarative tag resolved to a concrete resource, not a fork. The existing +`*_DEFAULT_CONNECTION` constants in `db.py` stay put; if CVL/Foundry ever migrate onto this path, +their tags join the registry then — and ideally the two registries share the tag namespace, so a +corpus's *import* target and its *runtime* search tools resolve by one name. + +--- + +## 5. Does the format generalize? (Foundry / CVL — future adopters, not now) + +No *existing* corpus moves onto this mechanism now — the first adopter is a new one (Crucible's). +But to be sure we aren't designing a Crucible-shaped format by accident, it's worth checking the +format could absorb the *other* corpora later. + +**Foundry** does real editorial grouping, not just parsing: it merges +`signature`/`description`/`parameters`/`returns` into **one** summary chunk keyed by the +cheatcode name, gives `Examples`/`Gotchas` their **own** chunks, and drops `Related Cheatcodes`. +All of that is producer layout, and its per-element decisions map 1:1 onto the block kinds: +parameter tables and lists are `atomic`, descriptions are `paragraph`s, follow-on prose is +`continuation`, samples are `code`. Its builder populates **only the vector index**, which the +format expresses directly as an `embedded_groups`-only manifest. A migration would also be the +moment to decide *deliberately* whether to start emitting `manual_sections`: Foundry's +`foundry_cheatcodes_keyword_search` / `..._get_section` tools currently query a +`manual_sections` table that nothing writes, and the dual declaration makes that gap visible in +the manifest instead of leaving it to ingestion side effects. + +**CVL** is the corpus that shows why the products must be independent: +[`ragbuild.py`](../composer/scripts/ragbuild.py) runs two pipelines over the same parsed HTML. +Its vector chunks are disjoint length-bounded passages built with exactly the per-element flags +the block kinds encode (paragraphs splittable, admonitions/lists/tables/asides atomic, stray +inter-tag text as continuation), while its manual documents are **subtree-inclusive** — a +section's document contains all its descendant sections inline, with editorial rules of its own +(an `Example` subsection never gets its own document). Flat sections feeding both indexes could +represent neither the overlap nor the flags; two independent products represent both directly. + +The genuinely corpus-specific pieces — Foundry's `.mdx → .html` conversion +([`foundry_process.py`](../composer/scripts/foundry_process.py)) and table-to-parameter-list +translation, CVL's docutils traversal — would stay in producers either way. So the model absorbs +both non-trivial corpora cleanly; the design isn't Crucible-only. + +--- + +## 6. What this gives Rust applications (the motivating case) + +Under the descriptor model a Rust app is a wheel + a declarative `AppDescriptor`; it already +names its corpus via `rag_db_default`. The missing piece is *contributing the corpus content* +without writing composer-resident Python glued to the RAG DB. Two levels: + +- **Level 1 (what this layer enables):** the app ships a `.rag.json` next to its crate (built + however it likes — a script in the app's own repo, checked-in output, a CI artifact) and the + generic `rag_import.py` ingests it. No checkout of the upstream doc source at build or run time. + Composer ships the importer and the schema, nothing corpus-specific. Crucible is the first app to + do this; see its own docs for that corpus. +- **Level 2 (optional, natural follow-on):** add a wheel FFI callout — `rag_entries() -> str` + returning the manifest JSON — so RAG content becomes part of the app package exactly like + `descriptor()`. The importer could then ingest straight from a loaded wheel + (`rag_import --from-wheel `), and a Rust app contributes a corpus with **zero** + Python. This is out of scope for v1 but is the reason the manifest is self-describing + (`knowledge_base` inside the document, not a CLI arg): a wheel can emit a complete, resolvable + corpus with no external metadata. + +--- + +## 7. What's built + +The mechanism, corpus-free: + +1. Manifest model [`composer/rag/import_format.py`](../composer/rag/import_format.py) — pydantic, + and deliberately importable with no RAG-stack dependency, so a producer needs only it. +2. The generic importer [`composer/scripts/rag_import.py`](../composer/scripts/rag_import.py) (§3), + covered by [`tests/test_rag_import.py`](../tests/test_rag_import.py) (each product feeds exactly + its own index, block kinds cut as declared, `part` numbering across sections *and* across + manifests sharing a DB, code-ref tagging, version and target-resolution refusals). +3. The `KNOWLEDGE_BASES` registry (§4) in [`composer/rag/db.py`](../composer/rag/db.py) — **empty**, + the same resting state as [`rag_env.py`](../composer/tools/rag_env.py)'s tools registry. + +What an adopting application adds, in one go: its committed `.rag.json`, both registry halves +(the `KNOWLEDGE_BASES` connection + a `composer/tools/_rag.py` in `rag_env._FACTORIES`), the +DB role/schema in [`init-db.sql`](../composer/scripts/init-db.sql), and whatever container wiring +populates it at `setup-db` time. + +**Untouched:** `foundry_ragbuild.py`, `ragbuild.py` (CVL), their wrappers, and `refresh_rag.sh`. +No runtime code changes — the search tools, `rag_env.py`, and the DB API are the same. + +--- + +## 8. Alternatives considered + +- **One shared section list feeding both indexes (an earlier draft of this format).** Rejected + once measured against what the two indexes actually store (§1): it forced the manual unit to + coincide with the vector grouping unit (CVL's subtree-inclusive documents are unrepresentable), + flattened every prose block to one kind so the importer had to guess chunking flags the + producer knew, and hard-wired "every section feeds both" so a vector-only corpus like Foundry's + was inexpressible. Replaced before any corpus shipped on it, so the schema version stayed 1. +- **Emit finished `BlockChunk`s in the JSON (cut below chunking).** Rejected: pushes spaCy + + the `max_length` policy + `` tagging + global `part` numbering into every + producer, re-duplicating the heavy, error-prone parts and re-coupling producers to the RAG + stack. The whole point is to keep producers dependency-free. +- **A plugin/entry-point registry of builders** (each corpus registers a `build()` callable) — + removes the shell duplication but keeps every builder coupled to the DB and the `ragbuild` + group, and gives Rust apps nothing (still composer-resident Python per corpus). A data format + is a stronger boundary than a code interface here: it's inspectable, diffable, cacheable, and + producible without the RAG stack. +- **Persist richer per-row metadata** (source URL, doc version, tags). Deferred: the current DB + schema is header-only, so v1 keeps provenance at the manifest level (logs only). Revisit with + a schema change if retrieval ever needs to filter on it. +- **One physical DB per corpus vs. one shared DB.** Orthogonal to this proposal — the + `KNOWLEDGE_BASES` registry expresses whatever the deployment already does (today: shared + `rag_db`, distinct roles; `extended_rag_db` separate). The format doesn't dictate topology. diff --git a/docs/rust-applications.md b/docs/rust-applications.md new file mode 100644 index 00000000..f2112662 --- /dev/null +++ b/docs/rust-applications.md @@ -0,0 +1,802 @@ +# Rust applications: the wheel API and the generic host + +How an AutoProver application whose backend is written in **Rust** is defined, and how the generic +Python host runs it. One Rust wheel plus the `AppDescriptor` it exports *is* the application: the +host synthesizes the phase enum, the CLI, the entry point, the frontend, the artifact store and +`main()` from that declaration, and drives the wheel's callouts through the shared pipeline. + +Reference for the seam as built. The driver it plugs into is +[formalization-abstraction.md](./formalization-abstraction.md) (`PipelineBackend` → +`PreparedSystem` → `Formalizer`); the confinement it runs its toolchain under is +[command-sandbox.md](./command-sandbox.md); the RAG corpus a wheel can declare is +[rag-import-format.md](./rag-import-format.md). + +--- + +## 1. What a Rust application is + +**Rust declares and decides; Python wires and does.** The wheel never owns an event loop, a DB +connection, a Textual widget or an `async with`. It contributes a descriptor (data) and callouts +(pure functions, plus two that spawn a subprocess and block). Python owns every imperative, +stateful, async edge: the LLM turns, the retry loop, Postgres, event streaming, caching, +confinement policy, the TUI. + +So the backend is a **passive service**, not a driver. The Python pipeline runs the +author→compile→judge→validate loop and calls the wheel when it needs an answer. Nothing in the +wheel holds state across calls, there is no resume/step protocol, and there is no async runtime in +Rust — no tokio, no `pyo3-async`, no `future_into_py`, no GIL-across-await marshalling. + +Two things are deliberately orthogonal: + +- **The backend's implementation language** — the wheel is compiled Rust. +- **The ecosystem**, i.e. the language of the *code being analyzed*. The wheel selects one by tag + (`descriptor.ecosystem` = `evm` | `solana` | `soroban`) and the host resolves it against + `composer.pipeline.ecosystem.ECOSYSTEMS`. The `echoprover` demo is a Rust wheel analyzing + Solidity. + +The ecosystem stays shared Python: the pipeline's front half (system analysis + property +extraction) is parametric over it, and a chain's system model, prompts and `locate_main` are +chain-specific, not app-specific — legitimately shared by every backend targeting that chain. The +line this draws: + +> Everything downstream of "which ecosystem" that is specific to **this verifier** lives in the +> wheel. Shared **service lifecycle** (Postgres, the TUI event loop, `composer.bind`) and shared +> **chain** logic (the ecosystem) stay Python. + +--- + +## 2. The FFI surface + +Eleven callouts, all **synchronous**, all speaking JSON strings. +[`export_app!`](../rust/autoprover-sdk/src/export.rs) generates every one of them from a `Backend` +impl. + +| Callout | Kind | Role | +| --- | --- | --- | +| `descriptor() -> str` | pure | the declarative spine (§3), read once at load | +| `validate_preconditions(args_json) -> str \| None` | pure | fail before any service opens; `None` = ok | +| `target_for(input_json, check) -> str \| None` | pure | which invocation a declared check runs under; `None` = its own (§6) | +| `author_prompt(input_json) -> str` | pure | the instruction (+ domain system prompt) for one authoring *session* (§5) | +| `check_syntax(input_json, spec) -> str \| None` | pure | reject a spec at write time; `None` = accept. Cheap — it runs on every put/edit | +| `judge(input_json) -> str \| None` | pure | who reviews this input's drafts; `None` = no judge. Asked once, before anything is authored | +| `judge_instruction(input_json, spec) -> str` | pure | what to ask that reviewer about this draft, per round (text, not JSON) | +| `compile(input_json, spec \| None, workdir, sandbox_json) -> str` | **blocking** | build the whole spec once — how setup and preflight build; `None` is the preflight, which has no spec | +| `validate(input_json, spec, target_json, workdir, sandbox_json) -> str` | **blocking** | build + check one target — which arrives with the rows it covers — returning a verdict per row (§6) | +| `workspace_prep(input_json) -> str` | pure | a *plan* the host executes (§7) | +| `sandbox_grants(args_json) -> str` | pure | extra grants to union into the host's policy (§8) | +| `finalize(outcomes_json) -> str \| None` | pure | run-level artifact files, `{relpath: contents}` (§9) | + +A callout that cannot produce its payload — bad input JSON, a serialize failure — returns +`{"kind":"error","message":…}` instead. The host raises before that string can be read as a +successful empty answer (no judge, no files, the check is its own target) or as a domain failure +the author should revise. `None` on an optional callout stays a successful empty answer. + +`compile` and `validate` run the real toolchain: each spawns `run-confined` and waits, for minutes. +They stay off the event loop without a bridge, by the pair that makes this whole design work — the +`#[pyfunction]` wraps its work in `py.allow_threads(…)`, and Python calls it with +`await asyncio.to_thread(...)` (via [`_blocking`](../composer/rustapp/session.py)). The wheel +just spawns and waits; Python moves the wait to a thread. That is also why the wheel spawns the +sandbox launcher *directly* rather than awaiting a Python runner: `run-confined` is a standalone +binary, so the wheel needs nothing from Python at run time except the policy data. + +### Where the ABI lives + +Every string crossing the boundary is a model in one of three places — two files holding the ABI +proper, plus the confinement wrapper, which belongs to the sandbox layer: + +- [descriptor.py](../composer/rustapp/descriptor.py) — the declarative half (`AppDescriptor` and + friends), mirroring the Rust structs. +- [wire.py](../composer/rustapp/wire.py) — the runtime half: `AuthorInput`, `Prompt`, `Failure`, + `Check`, `Verdict`, `CompileResult`, `ValidateOutcome`, `WorkspacePrep`, `SandboxGrants`, + `FinalizeInput`, and `CalloutError` (the envelope a callout returns instead of its payload). + Every `json.loads` of a wheel's answer happens in one of its `parse_*` functions, so a renamed + field fails at the boundary naming the field, not three frames later as an empty string. +- `BackendSpec` in [sandbox/config.py](../composer/sandbox/config.py) — the `sandbox_json` argument + the two blocking callouts receive, mirroring `autoprover_sdk::sandbox::Sandbox`. A `TypedDict` + rather than a model, because the sandbox layer deliberately carries no pydantic dependency; it is + held to the same round trip regardless, which is why `timeout_s` is bounded below (the mirrored + field is a `u64`, so a negative is refused at the wheel — the bound says so on this side too). + +Every payload that comes in more than one shape is a **tagged union** on both sides +(`#[serde(tag = …)]` ↔ pydantic discriminated union), so `CompileFailed` carries `errors` and no +verdicts, `ValidateVerdicts` the reverse, a `preflight` `AuthorInput` has neither a unit nor a +model, and a component that gave up carries nothing past its name. None of them can be asked for a +field another owns. + +Nothing on either side tolerates a missing or unexpected field — see **The strictness rule** below. + +Keeping the two halves in step is checked, not just asked for: +[test_wire_roundtrip.py](../tests/test_wire_roundtrip.py) round-trips every payload root through the +other language and back, in both directions, under Hypothesis. Its generator for the inbound +direction lives in Rust ([`autoprover_sdk::fuzz`](../rust/autoprover-sdk/src/fuzz.rs), behind the +`fuzz` feature, driven by the `wire-echo` binary over a pipe) rather than being derived from the +pydantic schema, because a generator built from the host's own models cannot produce a field only +the *Rust* side declares — the drift that matters most for an outbound payload, where it means the +wheel expects something the host never sends. + +A round trip is blind to one thing, but only on a *tolerant* seam: a field only one side declares, +which the other quietly defaults. Then "an older wheel omitted it" and "no wheel will ever send it" +are the same observation, and the field reads as `""` / `None` / an empty vec forever. + +This seam is not tolerant, so both cases fail at the callout that carries them, and the round trips +catch every class of drift on their own. `test_wire_roundtrip.py` keeps a deterministic per-type +field-set check beside them only so a one-sided field is named directly rather than surfacing as a +serde error inside a shrunk example. + +### The strictness rule + +The SDK and the host ship as a unit, so a payload missing a field is never version skew — it is a +mirror that drifted, and defaulting it converts a caught bug into a silent one. **The side that +deserializes requires everything.** Concretely, and in both directions: + +| | host → wheel | wheel → host | +| --- | --- | --- | +| a field only the *sender* declares | `#[serde(deny_unknown_fields)]` | `extra="forbid"` | +| a field only the *receiver* declares | no `#[serde(default)]` | no pydantic default | + +Two mechanical notes for anyone adding a field: + +- **`#[serde(default)]` on an `Option` does nothing.** serde deserializes a missing field of that + type as `None` on its own, whatever attributes are present, so the attribute reads as a deliberate + compatibility decision where none was made. Requiring such a field takes + [`crate::required::present`](../rust/autoprover-sdk/src/required.rs) via `deserialize_with`, which + serde cannot satisfy from an absent key. +- **Nothing carries `skip_serializing_if`.** An empty optional is spelled `null`, always present, so + absence is never a second way to say "nothing" and neither side has to treat the two alike. + +Two exceptions, both structural: [`AuthorInput`](../rust/autoprover-sdk/src/authoring.rs) cannot +carry `deny_unknown_fields` because serde rejects it alongside the `flatten` that `Authored` needs +(so a host-only field there is caught by the round trip rather than at the callout), and the outbound +*pydantic* models keep their defaults — Python only serializes those, `model_dump_json` writes every +field regardless, and an empty `source_unit` is how the host says it resolved none. + +`RustAppModule` is that surface as a Protocol, and `CALLOUTS` is derived from its annotations — so +[`load_module`](../composer/rustapp/host.py) rejects a module that isn't an AutoProver wheel (or is +one built against an older SDK) *at load*, naming the missing callouts, instead of dying with an +`AttributeError` several phases into a run. + +### Results, errors, and declining + +The pipeline result type (`FormT`) stays a Python pydantic model — +[`RustFormalResult`](../composer/rustapp/result.py) — because the driver's cache is keyed on it +(`cache_get(formalizer.formalized_type)` / `cache_put`) and must round-trip it. The wheel has no +result type of its own: it answers per target and the host accumulates. To *decline*, the host +returns the driver's `GaveUp` (a reportable outcome, not a crash); raised exceptions are reserved +for genuine failures the driver captures per component. + +--- + +## 3. The descriptor + +One struct, serialized at load time, that drives everything non-backend. +[Rust](../rust/autoprover-sdk/src/descriptor.rs) ↔ [Python](../composer/rustapp/descriptor.py). + +**Identity and vocabulary** + +| Field | Drives | +| --- | --- | +| `name` | the app name, task ids (`{name}-{step}`), the synthesized enum's class name | +| `header_text` | the TUI header | +| `ecosystem` | which `Ecosystem` the shared front half uses (default `evm`) | +| `backend_tag` | the report's backend vocabulary. Typed as `ReportBackend`, so a wheel declaring a tag the report doesn't know fails at descriptor load, before the run starts | +| `backend_guidance` | prose injected into the property-extraction prompt: what this verifier can check | +| `analysis_key` | the system-analysis cache key | +| `component_noun` | the human noun for one formalized component in the console/TUI ("instruction"); `None` → "component", read through `unit_noun()` | +| `check_noun` | what this backend calls one check **to the model** ("rule", "harness function"); `None` → "check", read through `check_label()` | +| `evidence_kinds` | the closed set an author may cite when rebutting the judge (§5) | + +**Phases.** `phases: [PhaseSpec { key, label, order, role }]`. The host resolves these once into a +`PhaseModel` ([`build_phase_model`](../composer/rustapp/host.py)): the synthesized +`enum.Enum(f"{Name}Phase", …)`, the driver's core-phase mapping, and the frontend's labels and +section order. Synthesizing the enum is safe because members are only ever used for `.name` and as +dict keys — nothing compares them against a static class. Resolving *once* is load-bearing, though: +`enum.Enum(...)` mints a fresh class per call, so labels keyed by one model's members are invisible +to another. Nothing else builds a model, and `build_backend` requires one rather than defaulting to +its own. + +A phase's `role` says **which step of the run it groups**: + +| `role` | The step | +| --- | --- | +| `analysis`, `extraction`, `formalization`, `report` | the four the driver itself tags. Required — `build_phase_model` raises unless every one is claimed | +| `discovery` | the design-doc task the *entry point* runs before the pipeline. Unclaimed, that task falls back to the first declared phase | +| `preflight` | the toolchain check (§4.2) | +| `setup` | the shared artifact authored before the fan-out (§4.3) | +| `grouping` (the default) | no step at all — a phase that only organizes the UI, like autoprove's harness/autosetup | + +Beyond the required four, **a role no phase claims is a step the application does not have**. For +`preflight` and `setup` — the steps the host runs as their own visible task — the phase is also the +declaration *of* that step: the task is the phase's own label, under the phase itself, with id +`{app}-{role}`. Claiming a step by role rather than by a side struct naming a `phase_key` means +there is no key for one half to spell and the other to match — and no way to point a step at a +phase that doesn't exist. + +**CLI.** `args: [ArgSpec { flag, help, default, required }]` become `add_argument` calls on top of +the three positional inputs (`project_root`, `main_contract`, `system_doc`) and the standard flags. +Their parsed values are threaded into `validate_preconditions` (as `AppArgs.declared`) *and* onto +every `AuthorInput.args`, so a knob like a fuzz budget reaches the wheel without a bespoke channel. + +**Events.** `event_kinds: [EventKind { kind, label, notice }]` tells the generic frontend how to +render each emitted payload (§10). A `notice` kind becomes a persistent callout plus a toast — for +one-shot important results such as a verdict — rather than a line in the collapsible log. + +**Layout.** `artifact_layout: ArtifactLayout` — `deliverable_dir`, `internal_dir`, `report_dir`, +`artifact_dir`, `artifact_prefix`, `artifact_extension`, `property_suffix`. + +**Steps and modes** — the fields that let a demanding app stay bespoke-Python-free: + +| Field | Effect | +| --- | --- | +| a phase with `role: preflight` | check the toolchain before authoring, as its own visible task (§4.2). No such phase: the workspace prep still runs, and nothing else changes — a wheel with nothing to check just doesn't declare one | +| a phase with `role: setup` | author one shared artifact before the per-component fan-out and hand it to every component as `AuthorInput.setup` (§4.3) | +| `deliverable_mode` | `per_component` (default), or `callout { primary? }` — where `primary` is the `{program}`-templated path used as each component's report link (§9) | +| `serialize_toolchain` | put the blocking callouts behind one `Semaphore(1)` — for an app sharing a single crate / target dir | +| `confine_by_default` | build the fail-closed `launcher` sandbox config by default (§8) | +| `rag_db_default` | the RAG corpus whose search tools the default env binds; validated at load (§10) | + +Everything but `name`/`header_text`/`backend_tag`/`backend_guidance`/`analysis_key`/`phases`/ +`artifact_layout` is defaulted, so a minimal wheel declares almost none of it — see +[example-app](../rust/example-app/src/lib.rs), which sets every step/mode field to its default and +is a complete application. + +--- + +## 4. The run, end to end + +```text +load ─ entry point ─┬─ preflight (workspace_prep → toolchain → build check) ─┐ + └─ system analysis ├─ prepare_system + ┘ + ─┬─ prepare_formalization ─────────────────┐ + └─ property extraction ───────────────────┴─ setup (author+compile, cached) ─ fan out + │ + per component: author ⇄ validate(target) → verdicts ───────────┘ + │ + report ─ store ─ finalize +``` + +The two build-shaped steps are overlapped with the LLM steps that don't need them, and the shared +setup spec is authored at the one point in the run where it can be (§4.3). + +### 4.1 Load and entry + +[`build_application`](../composer/rustapp/host.py) imports the module, validates the callouts, +parses the descriptor, resolves the ecosystem, validates the declared RAG corpus, synthesizes the +phase enum + core-phase map + labels + section order, and returns a `RustApplication`. Both +registry references are resolved up front: an unknown ecosystem or an unregistered corpus is a +wheel bug, not something to discover mid-run. + +[`rust_entry_point`](../composer/rustapp/entry.py) then does the irreducibly imperative half — +parse args, call `validate_preconditions` (with the run's inputs as `AppArgs` — the resolved +`source_unit`, so a wheel can check up-front that the code it will depend on is where the host +says it is, and the program/source path already split out of `path:Name`), open the four +Postgres-backed pools + RAG + async tool context + thread logger, build the `ServiceHost` env and +`WorkflowContext`, resolve or discover the design doc, apply `confine_by_default`, and yield the +Executor a frontend drives. + +### 4.2 Preflight: prepare the workspace, then check the toolchain + +[`RustBackend.preflight`](../composer/rustapp/adapter.py) runs **concurrently with system +analysis** — neither step reads the analyzed model, which is what makes the overlap safe — and the +driver cancels the analysis if it raises. It is created with `run.cpu_runner`, not `run.runner`: +the agent semaphore (`--max-concurrent`) budgets concurrent *agents*, and a multi-minute cargo +build charged to it would quietly take a quarter of the default concurrency away from the analysis +it overlaps. It is bounded by the run's other budget instead — the CPU semaphore +(`--max-cpu-tasks`, default 2), which is what keeps two toolchains off the same machine at once. + +Two steps, both *declared* by the wheel and executed here: + +1. **`workspace_prep`** (§7) — write the plan's files, then warm dependencies / build the program / + place its IDL through the chain's registered toolchain. Already the run's slowest non-LLM step. +2. **The toolchain check**, when the descriptor declares a `preflight` phase — an + `Authored::Preflight` `compile` whose `spec` is `None`, not an empty spec. Nothing has been + authored yet, so the wheel renders a throwaway skeleton of its own: the smallest artifact that + still exercises what an authored one will depend on. A wheel with nothing worth checking early + declares no such phase and stops after step 1. + +Step 1 alone is not enough, because `cargo fetch` resolves a dependency graph and **compiles +nothing** — and a failed warm is deliberately non-fatal. Skip the check and the first real build in +the run is draft #1's compile, at the far end of extraction — and an authoring agent cannot fix what +breaks there, because it does not own the manifest: + +| Failure | Where it would otherwise appear | +| --- | --- | +| Dependency graph won't co-resolve | compiler errors in draft #1 | +| The harness crate won't link (program on another Anchor major) | compiler errors in draft #1 | +| Codegen from the program rejects it | compiler errors in draft #1 | +| The built program isn't where the fixture expects, or won't load | a mystery panic at setup | + +So a failure is **terminal**: `run_preflight_gate` raises +[`PreflightFailed`](../composer/rustapp/adapter.py) with the diagnostics the wheel extracted, and +there is no re-author. Two side benefits: the check proves the built artifact actually runs, and it +leaves the target dir warm, so the first *authored* compile builds one crate instead of a graph. + +What the step establishes is carried forward as `ProjectFacts { source_unit, prep_facts }` — the driver +holds it opaquely and hands it to `prepare_system` — so the preflight build, every authoring turn and the +delivered artifact all agree on what they are building against. Both halves are chain-shaped (§7). + +### 4.3 Setup: the shared artifact + +A wheel that declares `setup` has one artifact every component builds on (a fixture, a shared +module). It must be authored from the **union of every component's properties** — that union is what makes them +checkable — which pins it to exactly one point in the run, and +[`prepare_formalization`](../composer/rustapp/adapter.py) is not it: that overlaps extraction, so no +properties exist there yet. Nor can it be lazy on first `formalize`: whichever component won the race +would decide the artifact all the others are then told to work within. + +So `prepare_formalization` returns a `StagedFormalizer` instead of a `Formalizer`, and the driver +calls [`RustStagedFormalizer.begin`](../composer/rustapp/adapter.py) between extraction and the +fan-out. `begin` takes every unit's properties unmerged, runs a `setup` session under the declared +step's task, and hands back the `RustFormalizer` built around the result — the artifact is threaded +through the constructor rather than assigned onto a live formalizer, so the formalizer is constructed +once and never mutated. + +Each wire `Property` names the unit it was inferred for (`Property.component`), because a title +identifies a property only *within* a unit — the two together are the report's `PropertyKey`. So two +units' same-titled properties are two different properties: both reach the artifact, the wheel can +tell them apart, and each is tied to the surface it has to be checkable against. On a component turn +the field is that turn's own unit; the setup turn is the one that sees more than one. + +A setup turn also carries the run's **unit set** (`Authored::Setup::units`). `begin` has it in hand at +this point, and it is the only callout that sees the set whole — a component turn holds one, a +preflight runs before any exists. Scaffolding for a multi-unit build is a function of the set rather +than of any one unit (a manifest's feature list, a crate root's module declarations), so a wheel whose +setup gate builds that scaffolding can build the real thing rather than a provisional shape something +later has to complete. It is passed for the gate's benefit, not the author's: nothing in the prompt +depends on it. + +The artifact is **cached** like a formalization result (`RustSetupSpec`), keyed by +[`_setup_identity`](../composer/rustapp/adapter.py): the program and its crate, the analyzed model, +the property set, and whether types come from the crate or a generated IDL. Deliberately not the +whole input — `args` also carries run knobs (a fuzz budget) that don't change what gets authored, and +`units` decides scaffolding rather than content, so a changed slug must not throw the artifact away. +Authoring + compiling this is a full LLM loop and on a large program the longest single step of a +run, so a re-run after a downstream failure must not pay for it twice. As with the driver's other +caches, changing the *prompt* does not invalidate; clear the namespace for that. + +A wheel with no `setup` gets its `RustFormalizer` directly from `prepare_formalization` and never +mentions staging. + +### 4.4 Formalization: the authoring session + +Per component, [`RustFormalizer.formalize`](../composer/rustapp/adapter.py) runs one session (§5). +Nothing about the checks is decided before it starts — the author decides them: + +```python +outcome = await run_session(module=…, input=…, kind="component", titles=…) +# inside the session, driven by the agent: +# put_spec / edit_spec → the buffer, gated by the wheel's check_syntax +# map_checks(mapping) → which checks verify which property. THIS is the work list: +# its distinct names are what runs, grouped by the wheel's target_for +# validate_spec(checks=None) → one run per DISTINCT target, each carrying the checks it covers; +# stamps the buffer's digest when every declared check is accounted +# for, and records what it covered as `ran` +# expect_check_failure(c,why) → a failure that IS the finding stops blocking the gate +# record_skip(p, why) → a property left out, with a reason; it must not be mapped +# feedback_tool(rebuttals=…) → the wheel's judge, structured; stamps on acceptance +# result(commentary) → refused unless every stamp matches the CURRENT buffer, and the +# declared mapping accounts for exactly what `ran` covered +# give_up(reason) → a real outcome, reported +``` + +The component path has **no separate `compile`**: `validate`'s build *is* the compile gate. Fusing +them is the efficiency win — a per-component dry-run before the first fuzz roughly doubled the e2e +— and it is why `ValidateOutcome` has a `BuildFailed` arm at all. Because the checks share one +build, a `BuildFailed` from any target fails the whole run and the author revises. + +`compile` is therefore called only for the two kinds that have no checks to validate: `setup` (whose +session is gated by `compile_spec`) and `preflight`. + +An authoring turn that produces nothing costs an attempt like any other, but the toolchain never +sees it — there is nothing to build — so the next prompt is told exactly that (`_NO_ARTIFACT`). + +### 4.5 Report and finalize + +`fetch_verdicts` maps the wire verdicts `validate` baked into the result onto the report's own +`Verdict` (its `message` is the wire `detail`), filling `unit_file` from the component when the +wheel didn't name one. The store writes the per-component artifacts and metadata (§9), and +`finalize` receives the whole outcome set as `FinalizeInput` — program, crate, IDL path, the shared +setup spec, and per component its `artifact_text`, `property_checks` and the `targets` its +checks ran under — and returns `{relpath: contents}` the host writes under the project root, +path-confined. + +--- + +## 5. Authoring and review + +Authoring is the **shared session** of [`composer/authoring/`](../composer/authoring/) — the same +workflow the CVL and foundry backends run, described in +[formalization-abstraction.md §4.3.1](./formalization-abstraction.md) — assembled for a wheel in +[`session.py`](../composer/rustapp/session.py). One stateful agent per spec: it owns a `curr_spec` +buffer, edits it, calls the gate, asks for review, and publishes. The wheel supplies the prompts and +answers the callouts; it is still a passive service. What follows is only what is Rust-specific. + +**The prompt is two halves.** The host owns the protocol half — the tools, what the publish gate +requires, what a skip and a give-up mean — rendered from +[`authoring_protocol.j2`](../composer/templates/authoring_protocol.j2). `Prompt.system` is the +*domain* half and is prepended to nothing else. + +**The wheel supplies its own noun.** Every prompt and tool description says what `check_noun` +declares — Crucible's author reads about *harness functions*, another wheel's about *invariants* — +because an author writes better when the prompt speaks the language its own generated code uses. The +tool *names* stay generic (`validate_spec`, `expect_check_failure`) so the protocol can name them +literally; only prose moves. The host renders this through `CheckVocab` plus graphcore +`tool_family`: the session tools declare `{check}` / `{checks}` placeholders and +`with_template` instantiates them per wheel. + +> `@tool_family_display` applies the UI label *after* `with_template`, so the generated +> subclass is what `as_tool`/`bind` close over. Putting `@tool_display` on the untemplated +> family class would rebind those methods over the *base* schema and the nouns would +> vanish with no error. `test_rust_llm_agent.py` guards it. A wheel that spelled the protocol itself could drift +from what the host enforces, so it is never asked to. The instruction is likewise augmented: the host +appends the obligation the gate enforces — declare the mapping before validating, cover every +property, and name only checks that are really in the spec — because that rule is the framework's +and identical for every backend, while the names themselves are the author's. + +**The gate is a tool, not a loop.** A component session gets `validate_spec` (the wheel's `validate`, +per target); a setup session gets `compile_spec` (the wheel's `compile`). A run that passes *stamps a +digest of the buffer it saw* into the session's validations, so any later edit invalidates it +without anything having to remember to clear it. A partial run — `validate_spec(checks=[…])`, for +iterating on one problem — never stamps. + +**A failure blocks unless it is the finding.** A check that did not come back `GOOD` keeps the gate +unstamped, unless the author marked it with `expect_check_failure(check, reason)`. That is how a real +counterexample reaches the report *as a finding with a justification* rather than as a row nobody +examined. It is the same mechanism as CVL's `expect_rule_failure` and foundry's +`expect_test_failure`. + +**The judge is structured.** When `judge` names a reviewer for an input, the session binds +`feedback_tool`; a wheel with no judge gets no review machinery and no feedback stamp among its +required validations. That question is asked once, when the session is built, and takes no spec — +whether an input is reviewed and who reviews it are both fixed before anything is authored, so only +`judge_instruction` is given a draft, once per round. The judge is a sub-agent that must read the draft +back through `get_spec` (`did_read`) and must call `result` with a `PropertyFeedback` — `good` is a +field it had to set, so there is no unparseable reply to interpret and no fail-open default. Its +acceptance is a stamp like any other. The author may answer a prior round with a `rebuttal`, typed +by the wheel's declared `evidence_kinds`. + +**Publishing** requires every stamp to match the current buffer and the property→checks mapping to +account for every property that was not skipped, checked against the checks the wheel declared. +`give_up(reason)` is the honest exit and is reported as a real outcome. + +--- + +## 6. Checks, targets and verdicts + +A **check** is a named, runnable verification in the authored artifact — a CVL rule, a foundry test, +a tagged fuzz assertion. It is what the report keys a row by, and it is the concept the whole seam is +organized around: a check yields a `Verdict`. (A *component* is the thing system analysis produced +and the pipeline fans out over; one component's session authors many checks.) + +**The author decides the check set.** Which checks express a component's properties — one rule +discharging three related invariants, two rules for one invariant — is authoring work, so it cannot +be computed before authoring starts. `map_checks` declares the property→checks mapping, and the +distinct names in it are exactly what a gate run executes. The relation is **many-to-many** and both +directions are ordinary: several checks under one property title, or one check named under several. + +A `Check` is `{ name, properties, target? }`. `properties` is the author's own claim, carried +verbatim from the mapping rather than guessed by the wheel — it is there because some checkers speak +in properties rather than in check names (Crucible tags each assertion message with its property +title, and that is what lets it place a counterexample), while a backend whose checker reports per +check can ignore it. A **target** is **one invocation of the checker** — one +build + one run — answered per name by the wheel's `target_for`. Several checks may share one, so a +wheel can put a component's whole property set in a single target. That is the run-vs-report split: +targets group what *runs*, checks group what is *reported*, and a target sits inside one component's +session, so the three nest (`component ⊇ target ⊇ check`). `target_for` returning `None` makes the +check its own target — one invocation per check, the default. + +A check's parts therefore come from the two parties that can know them: the **name** and **what it +verifies** from the author (the artifact is the author's, and so is the claim) and the **grouping** +from the wheel (a backend convention). The host puts them together, runs each *distinct* target once +(`target_or_name()`) and passes it as a `Target { name, checks }` carrying the checks it covers, so +the grouping is not something the wheel has to reconstruct; the wheel returns a verdict **per check +in it**. Attribution is the wheel's: it owns its result format, so it decides which check a +counterexample belongs to; the host records the verdicts verbatim and does no verdict logic of its +own, never parsing a tool's output. + +### The verdict contract + +Because the names are the author's, `validate` is also where a name is held to the artifact. A +backend must not answer `GOOD` for a check it found no evidence ran: + +| Situation | Verdict | +| --- | --- | +| The checker has no such check | `ERROR`, with a detail naming it | +| It exists but the run never exercised it | `UNKNOWN`, with a detail saying so | +| It ran and held | `GOOD` | +| It ran and was refuted | `BAD` + the counterexample | + +Both non-`GOOD` cases block the publish gate, which is the point: a declared check with nothing +behind it must not stamp a property as verified. What corroborates a check is the backend's own +business — a per-rule result from the Prover, a runtime tally of which tagged assertions a campaign +evaluated — but *some* evidence is required, and a scan of the source text is not it: a name in a +comment or in dead code reads exactly like a check. `Target::all(GOOD, …)` is therefore not a +legitimate answer to a clean run; "nothing was refuted" is one fact about the target, while `GOOD` +per check is a claim about each check individually. The residue no mechanism reaches — whether a +check that demonstrably ran genuinely verifies the property it claims — is the judge's. + +### Verdicts + +A verdict is keyed by **check name**, not by a restated `Check`. The wheel picks from the checks it +was just handed rather than echoing them back, and the host resolves each name to the `Check` it sent +before anything upstream sees it. Both ends hold that key to the target's own checks. On the Rust +side `ValidateOutcome::Verdicts` wraps a private `CheckVerdicts`, so the only ways to build one are +`Target::all` and `Target::verdicts`, which take the names from `self.checks` — a backend attributes +a run instead of spelling names. On the host side `ValidateVerdicts.resolve(target)` requires the +answer to be *exactly* the target's check set: a name no check has, a covered check left unanswered, +or the same check twice raises `ValidateCoverageError`. The unanswered case is the one worth the +machinery — a missing verdict is not a failing verdict, so it gives the publish gate nothing to +object to, and a wheel that answered for nothing would stamp a component nothing had checked. + +A stamping run records what it covered as `ran` — the targets, each with its checks — and that is +what the publish gate validates the declared mapping against, in both directions: every claimed name +must have run, and every name that ran must be claimed. Ground truth is the stamping run rather than +the current declaration, so a name added since is one that did not run and a name removed is one +that ran unclaimed; both are errors, which is why editing the mapping needs no stamp of its own. The +same `ran` reaches the result as `RustFormalResult.targets`, so a component's coverage stays +answerable even where a whole target erred. + +Verdicts are grouped by property, not appended as singletons — two checks verifying the same +property are two check names under one report row, and as singletons they would be two rows with the +same key that the store's `dict()` would silently collapse. + +`Outcome` is a closed enum on both sides (`GOOD` / `BAD` / `ERROR` / `TIMEOUT` / `UNKNOWN`) — the +report's backend-agnostic vocabulary, whose human wording ("No counterexample" vs "Verified") is +picked at render time from `backend_tag`, so a wheel never spells it out. A typo that used to reach +a report row as an unexplained `UNKNOWN` now doesn't compile. A label the *host* has never heard of +is refused rather than downgraded: with both sides shipping together it can only mean a variant added +to one `Outcome` and not the other, and rendering that as `UNKNOWN` would hide the drift behind a row +that looks merely inconclusive. `Verdict.detail` carries the counterexample or error text, so a bare +`BAD` is never unexplained. + +[`results.py`](../composer/rustapp/results.py) rolls these up for the console/TUI: one row per +*check*, with the tally in the report's own display order and wording. A row is named by the property +title when the check verifies exactly one, and otherwise by the check's own name — the only thing +that names it unambiguously once one check can carry several properties. A delivered component that +bakes no verdicts contributes one `UNKNOWN` row so the listing accounts for every component. + +--- + +## 7. The project seam + +Two things the host needs to know about a project it does not itself understand — where its code lives +as a unit of that project's build system, and how to prepare/build its workspace. Both are knowledge +about the **ecosystem under analysis**, not about implementing a backend in Rust: an application is +written in Rust, but the project it analyzes need not be. So +[toolchain.py](../composer/rustapp/toolchain.py) declares one seam, `ProjectToolchain`, and the +application that needs it registers an implementation per chain (shared by every wheel targeting it, +exactly like the ecosystem and RAG registries). + +**Everything project-shaped crosses that seam opaquely.** A Cargo package name, an Anchor IDL, a Move +package's named addresses are one ecosystem's vocabulary; a framework that declared fields for them +would make the next ecosystem an edit to [wire.py](../composer/rustapp/wire.py). So three payloads are +carried without a schema — `source_unit`, `prep_facts` and `WorkspacePrep.toolchain_request` (Rust: +`autoprover_sdk::chain::ChainData`, a JSON object and nothing more). They are typed at both *ends* and +nowhere in between: the chain's registered implementation and the wheels targeting that chain share +those types through the chain's own support crate — where that chain's Cargo/Anchor vocabulary and +its layout conventions live. Which type is inside follows from the wheel's declared `ecosystem`, not +from inspecting keys. It is the same treatment `AuthorInput`'s `model` and `unit` already get, for +the same reason. + +**`workspace_prep` is a pure plan the host executes.** The wheel returns +`WorkspacePrep { files, toolchain_request }` — file *contents* and declarative intent, never a command +line — and [`run_workspace_prep`](../composer/rustapp/adapter.py) writes the files itself +(path-confined) and hands the request to the chain's `ProjectToolchain`. The split is load-bearing for +the network posture: the sandbox **never** gives a confined process network access, so dependency +fetches run *unconfined* (a fetch executes no untrusted code) and anything that compiles runs +*confined + offline*. Handing the wheel a confined-with-network policy would be a brand-new security +capability; declaring a plan is not. + +What the toolchain establishes comes back as `prep_facts` on every later callout, so a fact there means +*the thing it describes is in place* — Solana's `{idl: }` is what routes a harness to generated +types instead of a dependency on a program crate it may not be able to link. A request that cannot be +carried out is a hard error: a wheel only asks when it cannot proceed without the result. + +`AuthorInput.program` is not part of any of this. It is only the *analysis* identifier (the `Name` in +`path:Name`) — a label and a namespace — and nothing about a build-system unit follows from it (a real +lending program: directory `programs/lend`, package `example-lending`, lib `example_lending`). That is +exactly why `source_unit` is resolved once and carried rather than derived per callout. + +**No chain has an entry today.** The seam's two halves are therefore reached in deliberately different +ways: + +- `source_unit` **degrades**. An empty answer is already a documented state — it is what Solidity + yields, and what an unreadable layout yields — and the wheel fills the gaps from its own convention + (`SolanaSourceUnit::resolved`). "No toolchain" is indistinguishable from "nothing to resolve", which + is the honest answer. +- `project_toolchain` **raises**. A plan that only places files never reaches it + (`WorkspacePrep.needs_toolchain`), so getting there means the wheel asked for preparation nothing can + perform. Skipping it silently would resurface much later as a mystifying compile error in the first + authored draft — read as the authoring agent's fault. + +--- + +## 8. Confinement and the security invariant + +> The LLM controls file **contents** only. The trusted wheel authors every argv. **Python** authors +> every sandbox policy. + +The wheel gets a `Workspace { dir, sandbox }` — the workdir and its `Sandbox { argv_prefix, +timeout_s }`, bundled because every command needs both — and launches +`[*argv_prefix, program, *args]` through the one shared helper, +[`Workspace::run`](../rust/autoprover-sdk/src/sandbox.rs). The prefix is **opaque**: +Python owns the confinement *intent* and lowers it to an argv (`SandboxConfig.backend_spec` → +`LauncherProvider.argv_prefix`), which names no sandbox mechanism, so swapping the mechanism never +changes this shape. An empty prefix is the trusted/`none` path — the command runs directly. A +non-empty one is a full `run-confined --` wrapper whose launcher confines *itself* +(Landlock + seccomp + rlimits + env allowlist) and then `execve`s the tool, fail-closed: + +```text +run-confined --rw --rw --ro --allow-env PATH … + --rlimit-as … -- + ^──────────── Python-authored ───────────^ ^─ wheel-authored ─^ +``` + +`Workspace::run` materializes the (possibly LLM-derived) `files` map into the workdir first, joining +each relative path through `confined_join` — rejecting absolute paths and `..` — and Landlock grants +only the `--rw` workdir, so even a bad path can't escape. The host's own writes (the prep plan's +files, `finalize`'s deliverables, an IDL a toolchain places) go through the mirror-image +[`confined_target`](../composer/rustapp/adapter.py). Timeouts are enforced in the helper (reader +threads avoid a pipe-buffer deadlock; a kill reports the captured output plus the timeout). + +A wheel that sets `confine_by_default` gets the fail-closed `launcher` provider by default, still +overridable by `COMPOSER_SANDBOX_PROVIDER`, with its declared `sandbox_grants` (`extra_ro` paths, +`extra_env` names) unioned into the Python-authored policy. + +Per seam: + +| Seam | New capability | Who authors argv | Who authors policy | +| --- | --- | --- | --- | +| `compile` / `validate` | runs the toolchain | the **wheel** (after `--`) | **Python** (before `--`) | +| `workspace_prep` | warm dirs, build a program, place an IDL | **Python** (the registered toolchain; the wheel supplies contents + which dirs/program) | **Python** | +| `finalize` / prep `files` | writes files under the project root | — (host writes, path-confined) | n/a | +| `sandbox_grants` | adds `extra_ro` / `extra_env` | n/a (data) | **Python** unions them in | + +No seam gives the LLM argv control, and none lets the wheel invent a policy. Full details in +[command-sandbox.md](./command-sandbox.md). + +--- + +## 9. Deliverables and the store + +[`RustArtifactStore`](../composer/rustapp/store.py) is a thin `ArtifactStore` subclass; the base +already writes everything identical across backends (`properties.json`, `commentary.md`, the +property→checks map, `token_usage.json`). All the subclass supplies is the descriptor's layout — and +the choice of how the *source* deliverable lands: + +- **`per_component`** (default) — one `{prefix}_{slug}.{ext}` file per component, written from its + `artifact_text` by the base writer. +- **`callout`** — the store writes **no** per-component source; it writes the shared metadata and + returns the mode's `primary` (`{program}`-templated) as the component's report link. The whole + deliverable comes from `finalize`, which is handed every component's `artifact_text`, + `property_checks` and `targets` plus the shared setup spec, and can therefore assemble one + artifact (a single crate with a section per property) as the single source of truth for its layout. + +The tradeoff of `callout` mode is that the deliverable lands on disk only at finalize, not +incrementally: the assembled artifact is only *runnable* once complete, and `validate` already +materializes a transient copy per run via the `files` map. Streaming partial deliverables would be a +deliberate follow-up. + +--- + +## 10. The Python shell: entry point, frontend, CLI + +All of this stays Python — it is service lifecycle and UI, not data — but it is +**descriptor-driven**, so an application supplies none of it. + +**Entry point** ([entry.py](../composer/rustapp/entry.py)). Irreducibly imperative async: nested +`async with` over `standard_connections` (checkpointer, store, pgvector indexed store, memory +backend), the async tool context and the thread logger; the `ServiceHost` env and +`WorkflowContext`; the design doc read through the async uploader; `import composer.bind` for its +import-time DI/tape bootstrap. What Rust contributes here is purely declarative: the arg schema and +the `validate_preconditions` hook. The env is descriptor-driven too — `build_default_env` binds the +standard source-navigation tools plus, when `rag_db_default` is set, that corpus's search tools +(none by default; the embedding stack is imported lazily so a wheel without a corpus never pays for +it). A backend wanting a different tool surface entirely passes `env_builder=`, which owns the whole +surface and takes no `rag_db`. + +**Frontend** ([frontend.py](../composer/rustapp/frontend.py)). Two thin subclasses of the shared +bases — a `MultiJobApp` TUI and a stdout handler — whose phase labels and section order come from the +descriptor. Domain-event rendering is data-driven: the gate tools emit with +[`emit_event`](../composer/rustapp/adapter.py), which puts `{"type": kind, …}` on the graph run's +custom stream, and the handler writes any *declared* kind to that task's +collapsible log, or, for a `notice` kind, posts a persistent callout with the report's own outcome +glyph. Rust controls the event *content*; Python does the emission (the wheel cannot call +`get_stream_writer()`), and no per-application handler subclass is needed. + +**CLI** ([cli.py](../composer/rustapp/cli.py)). Two `main()` shapes differing only in who owns the +event loop — `tui_main` (pipeline as a background worker inside Textual) and `console_main` +(pipeline directly, printing on completion). Both print the run summary, the counts block using +`unit_noun`, and the verdict tally + listing when the results carry verdicts (empty otherwise, so a +wheel that bakes none prints nothing). `import composer.bind` runs first. + +--- + +## 11. Packaging and writing a new application + +The wheel is its own maturin project, so `ai-composer` stays on setuptools and gains one dependency. +`requires-python = ">=3.12"` is a hard floor (the seam uses PEP 695 generics), so wheels are abi3 +for cp312+. + +1. **New crate** — `cdylib`, depending on `autoprover-sdk` and `pyo3` + (`features = ["extension-module", "abi3-py312"]`). The `[lib] name` MUST match the `export_app!` + module ident and the maturin module name. Copy + [example-app/Cargo.toml](../rust/example-app/Cargo.toml). +2. **Implement `Backend`** — `descriptor` + `author_prompt` + `compile` + `validate` are + required; `validate_preconditions`, `judge`, `judge_instruction`, `workspace_prep`, + `sandbox_grants` and `finalize` have defaults. Every callout is directly unit-testable in Rust with no Python. +3. **Export it** — `autoprover_sdk::export_app!(my_app, MyApp::new());` +4. **Wire the build** — a maturin `pyproject.toml` (`module-name = "my_app"`) with a `[tool.uv] + cache-keys` block over its `.rs` sources, then one line each in the root `pyproject.toml`'s + `apps` group and `[tool.uv.sources]`. `uv sync` builds it; there is no `maturin develop` step. +5. **Ship a CLI** — two lines, and register them under `[project.scripts]`: + + ```python + from composer.rustapp.cli import console_main, tui_main + def main() -> int: return tui_main("my_app") + def main_console() -> int: return console_main("my_app") + ``` + + For headless/programmatic use: `await run_rust_pipeline("my_app", source, ctx, handler, env)`. + +Two escape hatches exist for an app the descriptor cannot express, and neither is needed by an app +that fits it: `build_application(store_factory=…, backend_cls=…)` for a specialized store or +prepared-system path, and `env_builder=` for a bespoke tool surface. + +--- + +## 12. Current limits + +Facts about the seam as it stands, not open design questions: + +- **Validation is serial.** `validate` is per-target and the host owns scheduling, so fanning out + with `asyncio.gather` is a Python-side change with no API impact — but a wheel sharing one crate + hits binary-name collisions, which is what `serialize_toolchain` exists for. Real parallelism + needs the wheel to separate build from run first. +- **No registered chain implementations.** `SOURCE_CRATES` and `WORKSPACE_TOOLCHAINS` are both + empty here (§7): a files-only prep plan works, a plan asking for a warm/build/IDL raises. +- **Self-contained backends only.** This shape fits a checker that is a **local tool**. A backend + whose "validate" is a remote/Python service cannot spawn it under `run-confined`; there is no + `run_prover`-style host effect, so such a backend would need one added (or stays a Python + backend). +- **No HITL.** The generic task handler raises on an interrupt prompt; an interactive Rust + application would need a new mechanism. +- **No build-only tool in the component belt.** `validate` fuses build and run, so a separate + `check_build` would only save the checker's runtime on a draft that does not compile. Worth adding + if a real run shows an author burning its fuzz budget on build errors — the callout is already + wrapped for the setup session's `compile_spec`. +- **Undeclared work is invisible.** The check set is the author's declaration, so a check written + but left out of the mapping simply never runs. The gate catches the reverse (a name that did not + run) but has nothing to compare against for work never declared. +- **Prompt changes don't invalidate caches** — neither the setup spec's nor the driver's. Clear + the namespace. + +--- + +## 13. Key files and tests + +| Concern | File | +| --- | --- | +| The SDK: the `Backend` trait, descriptor, wire types, `Workspace::run`, `export_app!` | [rust/autoprover-sdk/src/](../rust/autoprover-sdk/src/) | +| A complete minimal application | [rust/example-app/src/lib.rs](../rust/example-app/src/lib.rs) | +| The sandbox launcher | [rust/run-confined/src/main.rs](../rust/run-confined/src/main.rs) | +| Declarative ABI mirror | [composer/rustapp/descriptor.py](../composer/rustapp/descriptor.py) | +| Runtime ABI mirror + parsers | [composer/rustapp/wire.py](../composer/rustapp/wire.py) | +| The backend, preflight, prep, report | [composer/rustapp/adapter.py](../composer/rustapp/adapter.py) | +| The authoring session (buffer, gate, review, publish) | [composer/rustapp/session.py](../composer/rustapp/session.py) | +| The shared authoring workflow | [composer/authoring/](../composer/authoring/) | +| Application assembly (enum, phases, store, backend) | [composer/rustapp/host.py](../composer/rustapp/host.py) | +| Entry point / argparse / env | [composer/rustapp/entry.py](../composer/rustapp/entry.py) | +| Frontend, CLI, verdict rollup, store, result | [frontend.py](../composer/rustapp/frontend.py) · [cli.py](../composer/rustapp/cli.py) · [results.py](../composer/rustapp/results.py) · [store.py](../composer/rustapp/store.py) · [result.py](../composer/rustapp/result.py) | +| Chain seams (crate resolution, workspace toolchain) | [composer/rustapp/toolchain.py](../composer/rustapp/toolchain.py) | +| The driver this plugs into | [composer/pipeline/core.py](../composer/pipeline/core.py) | +| Sandbox policy authoring | [composer/sandbox/](../composer/sandbox/) | + +Tests: `tests/test_rustapp.py` (the wheel round-trip, end to end through the host), +`test_rustapp_wire.py` (ABI), `test_wire_roundtrip.py` (both directions of the ABI under +Hypothesis, against the real serde types), `test_rustapp_preflight.py`, `test_rustapp_workspace_prep.py`, +`test_rustapp_setup_cache.py`, `test_rustapp_verdicts.py`, `test_rustapp_toolchain_sem.py`, +`test_rustapp_gate.py`, `test_rustapp_validate_target.py`, `test_rustapp_discovery_phase.py`, +`test_rust_llm_agent.py`, +`test_rust_frontend.py`, plus `test_sandbox_run_confined.py` / `test_sandbox_escape.py` for the +launcher contract. diff --git a/graphcore b/graphcore index 5b295139..54a6852e 160000 --- a/graphcore +++ b/graphcore @@ -1 +1 @@ -Subproject commit 5b295139242ba81ecc98ef986deb0ea31a240b68 +Subproject commit 54a6852eac72896d17d3bc4f3068a775f9200d1f diff --git a/pyproject.toml b/pyproject.toml index 70757838..dee16df6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,5 +1,11 @@ [build-system] -requires = ["setuptools>=61.0", "wheel"] +# Two constraints, the second binding. The ``templates/**/*.j2`` entry under +# [tool.setuptools.package-data] needs 62.3+, which is where package-data globs +# gained ``recursive=True``; below that ``**`` collapses to a single ``*`` and +# every top-level template silently drops out of the wheel. But 67.0 is the +# floor that actually applies: earlier releases import ``pkgutil.ImpImporter``, +# removed in 3.12, so they cannot run on the Python this project requires. +requires = ["setuptools>=67.0", "wheel"] build-backend = "setuptools.build_meta" # please let me land this stupid PR @@ -17,7 +23,7 @@ authors = [ ] dependencies = [ - "graphcore @ git+ssh://git@github.com/Certora/graphcore.git@5b295139242ba81ecc98ef986deb0ea31a240b68", + "graphcore @ git+ssh://git@github.com/Certora/graphcore.git@54a6852eac72896d17d3bc4f3068a775f9200d1f", "aiohttp>=3.13", "attrs>=26.1", "Jinja2>=3.1", @@ -67,7 +73,7 @@ s3 = ["s3fs>=2023.1.0"] # exactly one extra must be selected at install time (enforced via the # [tool.uv] conflicts block below). Each extra is named after the package it # pulls. -certora-cli = ["certora-cli>=8.18.0"] +certora-cli = ["certora-cli>=8.19.1"] certora-cli-beta = ["certora-cli-beta>=8.18.0"] certora-cli-beta-mirror = ["certora-cli-beta-mirror>=8.18.0"] # Back-compat alias: `prover` == stable channel. @@ -95,8 +101,43 @@ ragbuild = [ "markdown-it-py>=3.0", "mdit-py-plugins>=0.4", ] +dev = [ + "maturin>=1.14.1", + # `dev` is one of uv's default groups, so pulling `apps` in here means a bare + # `uv sync` builds the Rust artifacts — while the container image, which sets + # UV_NO_DEV=1, still gets neither (its final stage has no Rust toolchain). + { include-group = "apps" }, +] +# Everything built out of the `rust/` workspace, declared as path deps below so +# `uv sync` builds AND preserves them (a bare sync prunes hand-built wheels as +# extraneous). Kept OUT of [project.dependencies] and reachable only via `dev` so +# an install without cargo stays possible; each project carries `[tool.uv] +# cache-keys` over its `.rs` sources, so uv rebuilds on `uv run` when the Rust +# changes and there is no manual `maturin develop` / `cargo build` step. +apps = [ + # crucible_app is added in PR3 (its crate lives in rust/crucible-app, which lands there). + "echoprover", + # The command-sandbox launcher — a binary, not an extension module; the wheel + # lands it in `.venv/bin`, which is where composer.sandbox.launcher resolves it + # from. Landlock/seccomp are Linux-only, hence the marker. + "run-confined; sys_platform == 'linux'", + # Optional belt-and-braces: recompiles a changed crate at *import* time, which + # also covers running `python` directly in an activated venv (no `uv run`). + # Redundant with the cache-keys above; activate per venv if you want it: + # python -m maturin_import_hook site install + "maturin-import-hook>=0.3.0", +] [tool.uv] +# The `apps` group builds several maturin crates. On a machine that does not have the +# pinned toolchain yet, each one's cargo call asks rustup to install it, and rustup's +# install path is not safe against concurrent invocations — whichever call wins clears +# $RUSTUP_HOME/downloads and the others die renaming a half-downloaded component. +# Costs next to nothing: the crates share one cargo workspace, so their builds already +# serialize on rust/target's build-dir lock. This only covers builds uv itself drives; +# installing the toolchain first (see README) also covers rust-analyzer and a second +# terminal. +concurrent-builds = 1 conflicts = [ [{ extra = "cpu" }, { extra = "cuda" }], [ @@ -108,6 +149,12 @@ conflicts = [ [tool.uv.sources] graphcore = { path = "./graphcore", editable = true } +# Wheels built from the local Rust workspace (see the `apps` group above). +# crucible_app's source is added in PR3 (with its crate under rust/crucible-app). +echoprover = { path = "rust/example-app", editable = true } +# Not editable: a bin wheel has nothing to link back to a source tree — uv rebuilds +# it from the cache-keys in rust/run-confined/pyproject.toml instead. +run-confined = { path = "rust/run-confined" } torch = [ { index = "pytorch-cpu", extra = "cpu" }, { index = "pytorch-cu128", extra = "cuda" }, @@ -126,7 +173,8 @@ explicit = true [tool.pytest.ini_options] markers = [ "expensive: hits the real prover / network — deselect with -m 'not expensive'", - "fuzz: fuzz tests" + "fuzz: fuzz tests", + "wire: Rust/Python wire-protocol round trips — build a cargo binary at test time" ] # optional but recommended once you're registering marks: addopts = "--strict-markers" # a typo'd mark becomes an error, not a silent no-match @@ -139,6 +187,9 @@ console-autoprove = "composer.cli.console_autoprove:main" tui-autoprove = "composer.cli.tui_autoprove:main" console-foundry = "composer.cli.console_foundry:main" tui-foundry = "composer.cli.tui_foundry:main" +# console-crucible / tui-crucible land in PR3, together with the `composer.crucible_launch` +# module they name — an entry point pointing at a module that doesn't exist installs happily +# and only fails, as an ImportError, when someone runs it. autoprove-report-render = "composer.spec.source.report.render:main" tui-natspec = "composer.cli.tui_pipeline:main" cache-natspec = "composer.cli.cache_natspec:main" @@ -155,7 +206,11 @@ certora-fixconf = "certora_autosetup.fixconf:main" autosetup = "certora_autosetup.autosetup.cli:main" fixconf = "certora_autosetup.fixconf:main" -# Command-sandbox providers, discovered by composer.sandbox.SandboxConfig.resolve_provider. +# summarization_detector entry points +detect-summaries = "summarization_detector.cli:main" +detect-difficulty-profile = "summarization_detector.difficulty_profile:main" + +# Command-sandbox providers, resolved by composer.sandbox.config.SandboxConfig.resolve_provider. # Adding a provider = adding a line here (and re-running the editable install so the # dist-info picks it up); the seam never imports a concrete mechanism itself. [project.entry-points."composer.sandbox_providers"] @@ -163,13 +218,16 @@ none = "composer.sandbox.policy:NoneProvider" launcher = "composer.sandbox.launcher:LauncherProvider" [tool.setuptools.packages.find] -include = ["composer*", "sanity_analyzer*", "analyzer*", "certora_autosetup*"] +include = ["composer*", "sanity_analyzer*", "analyzer*", "certora_autosetup*", "summarization_detector*"] [tool.setuptools.package-dir] "" = "." [tool.setuptools.package-data] -composer = ["templates/**/*.j2", "scripts/init-db.sql", "kb/resources/**/*.md", "kb/resources/*.yaml"] +composer = [ + "templates/**/*.j2", "templates/*.js", "scripts/init-db.sql", + "kb/resources/**/*.md", "kb/resources/*.yaml", +] # Bundled CVL summaries, spec templates, mocks and conf templates the prover # reads directly off disk; ship them with the wheel. certora_autosetup = [ diff --git a/pyrightconfig.json b/pyrightconfig.json index bd24c4d8..d19fe82c 100644 --- a/pyrightconfig.json +++ b/pyrightconfig.json @@ -1,5 +1,5 @@ { - "include": ["composer", "analyzer", "certora_autosetup", "sanity_analyzer"], + "include": ["composer", "analyzer", "certora_autosetup", "sanity_analyzer", "summarization_detector"], "extraPaths": ["graphcore"], "stubPath": "stubs", "typeCheckingMode": "standard", diff --git a/rust-toolchain.toml b/rust-toolchain.toml new file mode 100644 index 00000000..f4096656 --- /dev/null +++ b/rust-toolchain.toml @@ -0,0 +1,16 @@ +# Toolchain for the `rust/` workspace (PyO3 app wheels + the run-confined launcher). +# +# rustup reads this by walking UP from the *current directory* — it ignores +# `--manifest-path`. Since cargo gets invoked both from a crate dir (maturin +# building `rust/example-app`) and from the repo root +# (`cargo build -p run-confined --manifest-path rust/Cargo.toml`, as the sandbox +# VM provisioner does), the file lives at the repo root so both resolve to it. +# +# rustup installs a missing toolchain automatically on the first cargo call, so a +# contributor needs only rustup — no version to pick by hand. To bump, change +# `channel` (patch releases resolve automatically: "1.96" -> 1.96.x). +[toolchain] +channel = "1.96" +profile = "minimal" +# minimal drops rustfmt/clippy; anyone editing these crates wants both. +components = ["rustfmt", "clippy"] diff --git a/rust/.gitignore b/rust/.gitignore new file mode 100644 index 00000000..8c8c96b5 --- /dev/null +++ b/rust/.gitignore @@ -0,0 +1,5 @@ +/target +**/*.rs.bk +# maturin / local build venvs +.venv/ +*.whl diff --git a/rust/Cargo.lock b/rust/Cargo.lock index e5575252..25cb86d6 100644 --- a/rust/Cargo.lock +++ b/rust/Cargo.lock @@ -52,6 +52,38 @@ dependencies = [ "windows-sys", ] +[[package]] +name = "arbitrary" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3d036a3c4ab069c7b410a2ce876bd74808d2d0888a82667669f8e783a898bf1" +dependencies = [ + "derive_arbitrary", +] + +[[package]] +name = "autocfg" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" + +[[package]] +name = "autoprover-sdk" +version = "0.1.0" +dependencies = [ + "arbitrary", + "libc", + "pyo3", + "serde", + "serde_json", +] + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + [[package]] name = "clap" version = "4.6.4" @@ -98,6 +130,17 @@ version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" +[[package]] +name = "derive_arbitrary" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e567bd82dcff979e4b03460c307b3cdc9e96fde3d73bed1496d2bc75d9dd62a" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + [[package]] name = "enumflags2" version = "0.7.12" @@ -115,7 +158,17 @@ checksum = "67c78a4d8fdf9953a5c9d458f9efe940fd97a0cab0941c075a813ac594733827" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", +] + +[[package]] +name = "example-app" +version = "0.1.0" +dependencies = [ + "autoprover-sdk", + "pyo3", + "serde", + "serde_json", ] [[package]] @@ -124,17 +177,32 @@ version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" +[[package]] +name = "indoc" +version = "2.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "79cf5c93f93228cf8efb3ba362535fb11199ac548a09ce117c9b1adc3030d706" +dependencies = [ + "rustversion", +] + [[package]] name = "is_terminal_polyfill" version = "1.70.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + [[package]] name = "landlock" -version = "0.4.5" +version = "0.4.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "635839550ae8b90d9fd2571460a6645dc0aec070225956ca7a2831ed31d2795d" +checksum = "4cca98e95f35b29d469dade6724c6f96cec9236640f745a0e99b0334ec320ab1" dependencies = [ "enumflags2", "libc", @@ -143,9 +211,30 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.186" +version = "0.2.189" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" + +[[package]] +name = "memchr" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" + +[[package]] +name = "memoffset" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "488016bfae457b036d996092f6cb448677611ce4449e970ceaf42695203f218a" +dependencies = [ + "autocfg", +] + +[[package]] +name = "once_cell" +version = "1.21.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "once_cell_polyfill" @@ -153,20 +242,89 @@ version = "1.70.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" +[[package]] +name = "portable-atomic" +version = "1.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d20d5497ef88037a52ff98267d066e7f11fcc5e99bbfbd58a42336193aacec3" + [[package]] name = "proc-macro2" -version = "1.0.106" +version = "1.0.107" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" dependencies = [ "unicode-ident", ] +[[package]] +name = "pyo3" +version = "0.24.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5203598f366b11a02b13aa20cab591229ff0a89fd121a308a5df751d5fc9219" +dependencies = [ + "cfg-if", + "indoc", + "libc", + "memoffset", + "once_cell", + "portable-atomic", + "pyo3-build-config", + "pyo3-ffi", + "pyo3-macros", + "unindent", +] + +[[package]] +name = "pyo3-build-config" +version = "0.24.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "99636d423fa2ca130fa5acde3059308006d46f98caac629418e53f7ebb1e9999" +dependencies = [ + "once_cell", + "target-lexicon", +] + +[[package]] +name = "pyo3-ffi" +version = "0.24.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78f9cf92ba9c409279bc3305b5409d90db2d2c22392d443a87df3a1adad59e33" +dependencies = [ + "libc", + "pyo3-build-config", +] + +[[package]] +name = "pyo3-macros" +version = "0.24.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b999cb1a6ce21f9a6b147dcf1be9ffedf02e0043aec74dc390f3007047cecd9" +dependencies = [ + "proc-macro2", + "pyo3-macros-backend", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "pyo3-macros-backend" +version = "0.24.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "822ece1c7e1012745607d5cf0bcb2874769f0f7cb34c4cde03b9358eb9ef911a" +dependencies = [ + "heck", + "proc-macro2", + "pyo3-build-config", + "quote", + "syn 2.0.119", +] + [[package]] name = "quote" -version = "1.0.46" +version = "1.0.47" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dfbc457d0c7a0759a614551b11a6409e5951f6c7537be1f1b7682b9ae9230368" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" dependencies = [ "proc-macro2", ] @@ -181,6 +339,12 @@ dependencies = [ "seccompiler", ] +[[package]] +name = "rustversion" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f" + [[package]] name = "seccompiler" version = "0.5.0" @@ -190,6 +354,49 @@ dependencies = [ "libc", ] +[[package]] +name = "serde" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.3", +] + +[[package]] +name = "serde_json" +version = "1.0.151" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + [[package]] name = "strsim" version = "0.11.1" @@ -198,9 +405,9 @@ checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" [[package]] name = "syn" -version = "2.0.118" +version = "2.0.119" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b9ae57f904213ebb649ce6895b8a66c66f0203b9319718f69a5612a065b1422" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" dependencies = [ "proc-macro2", "quote", @@ -218,24 +425,30 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "target-lexicon" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "adb6935a6f5c20170eeceb1a3835a49e12e19d792f6dd344ccc76a985ca5a6ca" + [[package]] name = "thiserror" -version = "2.0.18" +version = "2.0.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4288b5bcbc7920c07a1149a35cf9590a2aa808e0bc1eafaade0b80947865fbc4" +checksum = "09a43598840e33d5b0331f38c5e30d13bb11c11210a4b58f0d9b18a5a5eefcd9" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "2.0.18" +version = "2.0.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5" +checksum = "43cbfe0cf76104d42a574802844187e84a305e531ed54455f11fbde0f10541cd" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn 3.0.3", ] [[package]] @@ -244,6 +457,12 @@ version = "1.0.24" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" +[[package]] +name = "unindent" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7264e107f553ccae879d21fbea1d6724ac785e8c3bfc762137959b5802826ef3" + [[package]] name = "utf8parse" version = "0.2.2" @@ -264,3 +483,9 @@ checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" dependencies = [ "windows-link", ] + +[[package]] +name = "zmij" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" diff --git a/rust/Cargo.toml b/rust/Cargo.toml index be0ad6b3..ce41e184 100644 --- a/rust/Cargo.toml +++ b/rust/Cargo.toml @@ -1,21 +1,28 @@ -# Cargo workspace for AutoProver's Rust components. +# Cargo workspace for AutoProver's Rust extension framework. # -# In this PR the workspace contains a single member: +# * `autoprover-sdk` — the library new Rust-based applications import. It defines +# the ABI (serde types), the `Backend` trait, the `run_confined` launcher helper, +# the FFI helpers, and the `export_app!` macro that emits the PyO3 module. +# * `example-app` — a self-contained demonstration application built into a +# Python wheel via maturin, used by the framework's round-trip test. # -# * `run-confined` — the trusted command-sandbox launcher (Landlock filesystem + -# seccomp network/ptrace + rlimits + scrubbed env, then `execve`). The first -# `SandboxProvider` for the `RunCommand` effect — see docs/command-sandbox.md. -# -# Later PRs re-add the Rust *application framework* crates to `members`: -# `autoprover-sdk` (the ABI + `export_app!` macro) and `example-app` (its -# round-trip test wheel), then the `crucible-app` wheel. Those crates depend on -# the shared `[workspace.dependencies]` (serde / pyo3); `run-confined` does not, -# so this trimmed workspace declares none. +# A new application is its own crate (cdylib) that depends on `autoprover-sdk` +# and invokes `autoprover_sdk::export_app!`. It lives outside this workspace in +# real use; `example-app` is kept here so the framework has something to test. [workspace] resolver = "2" -members = ["run-confined"] +members = ["autoprover-sdk", "example-app", "run-confined"] [workspace.package] edition = "2021" version = "0.1.0" license = "MIT" + +[workspace.dependencies] +serde = { version = "1", features = ["derive"] } +serde_json = "1" +pyo3 = "0.24" +# Compile-time Jinja-style templates (the `.j2` convention used by composer/templates/*.j2, +# here for the Rust prompt/crate-file templates). Templates are checked at build time and +# embedded in the wheel, so no runtime template loading or packaging. +askama = "0.13" diff --git a/rust/README.md b/rust/README.md new file mode 100644 index 00000000..c56054eb --- /dev/null +++ b/rust/README.md @@ -0,0 +1,120 @@ +# AutoProver Rust framework + +Build AutoProver formalization backends / applications in Rust and run them +through the generic Python pipeline via PyO3. Design rationale and the full seam: +[docs/rust-applications.md](../docs/rust-applications.md). + +## Layout + +| Crate | Role | +| --- | --- | +| [`autoprover-sdk`](autoprover-sdk) | The library a Rust application imports: the ABI (serde types), the `Backend` trait, the `run_confined` launcher helper, the FFI helpers, and the `export_app!` macro. | +| [`example-app`](example-app) | The `echoprover` demo — a complete, self-contained application built into a wheel and exercised by `tests/test_rustapp.py`. | +| [`run-confined`](run-confined) | The command-sandbox launcher (Landlock + seccomp). Not an extension module: it builds as a maturin *bin* wheel, so `uv sync` lands the binary in `.venv/bin`. See [docs/command-sandbox.md](../docs/command-sandbox.md). | + +## Building + +Nothing here is built by hand. Every crate that ships is a `uv` path dependency in the +root `pyproject.toml`'s `apps` group (reachable from the default `dev` group), so: + +```sh +uv sync # builds echoprover + run-confined into the venv +uv run pytest tests/test_rustapp.py tests/test_sandbox_launcher.py +``` + +Each project declares `[tool.uv] cache-keys` over its `.rs` sources, so uv rebuilds a +crate on the next `uv run` after you edit it — there is no `maturin develop` step. The +toolchain is pinned in [rust-toolchain.toml](../rust-toolchain.toml) and rustup installs +it on demand. The container image sets `UV_NO_DEV=1` and so builds none of this (its +final stage has no cargo). + +The Python side is [`composer/rustapp`](../composer/rustapp): it loads a wheel, +synthesizes the pipeline's phase enum from the descriptor, and drives the wheel — +a **passive service** — through the author→compile→judge→validate loop. Python owns +the loop, every LLM turn, and all async I/O; the wheel answers pure questions and +runs its own toolchain. No `pyo3-async` bridge is involved. + +## The FFI surface + +A wheel exports exactly (all synchronous, JSON strings across the boundary): + +```text +descriptor() -> str # the AppDescriptor +validate_preconditions(args_json) -> str|None +target_for(input_json, check) -> str|None # the invocation a declared check runs under +author_prompt(input_json) -> str # one authoring session's prompt +check_syntax(input_json, spec) -> str|None # None ⇒ the spec may be written +judge(input_json) -> str|None # None ⇒ no judge for this input +judge_instruction(input_json, spec) -> str # one review round's instruction +compile(input_json, spec|None, workdir, sandbox_json) -> str # BLOCKING (run-confined) +validate(input_json, spec, target, workdir, sandbox_json) -> str # BLOCKING (run-confined) +workspace_prep(input_json) -> str # a plan the host executes +sandbox_grants(args_json) -> str +finalize(outcomes_json) -> str|None +``` + +`export_app!` generates all of these. `compile`/`validate` release the GIL while their +child process runs, so the host calls them with `asyncio.to_thread`. A callout that cannot +produce its payload returns `{"kind":"error","message":…}` instead of a fake success. + +## Writing a new application + +1. New crate: `cdylib`, depending on `autoprover-sdk` and `pyo3` + (`features = ["extension-module", "abi3-py312"]`). See + [example-app/Cargo.toml](example-app/Cargo.toml). + +2. Implement `Backend`. Required: `descriptor` + `author_prompt` + `compile` + + `validate`. Defaulted: `target_for`, `validate_preconditions`, `check_syntax`, + `judge`, `judge_instruction`, `workspace_prep`, `sandbox_grants`, `finalize`. See + [example-app/src/lib.rs](example-app/src/lib.rs). + +3. Export the module (ident must match the wheel/module name): + + ```rust + autoprover_sdk::export_app!(my_app, MyApp); + ``` + +4. Add a maturin `pyproject.toml` (`module-name = "my_app"`) with a `[tool.uv] + cache-keys` block over its `.rs` sources — copy + [example-app/pyproject.toml](example-app/pyproject.toml). Then wire it into the root + `pyproject.toml` (one line in the `apps` group, one in `[tool.uv.sources]`) and + `uv sync`; there is no separate build command. + +5. Ship a CLI. The generic entry point + frontend are synthesized from the + descriptor — a runnable app is a two-line `main()`: + + ```python + # my_app_cli.py + from composer.rustapp.cli import tui_main, console_main + def main() -> int: return tui_main("my_app") # Textual TUI + def main_console() -> int: return console_main("my_app") # stdout + ``` + + Register them in `pyproject.toml` under `[project.scripts]`. No bespoke + argparse, entry point, frontend, or `main()` to write — the descriptor drives + all of it (CLI flags, precondition validation, phase labels, event rendering, + artifact layout). + + For programmatic / headless use, the pipeline wrapper is also exposed directly: + + ```python + from composer.rustapp import run_rust_pipeline + result = await run_rust_pipeline("my_app", source_input, ctx, handler_factory, env) + ``` + +## Testing the demo + +```sh +uv run pytest tests/test_rustapp.py # `uv sync` already built the echoprover wheel +``` + +## Notes + +* `Backend` is `Send + Sync + 'static` (one instance per wheel, built once in + `export_app!`); holding no per-call state satisfies this without effort. +* `compile`/`validate` author their own command line but never their own + confinement: they prepend the opaque `Sandbox::argv_prefix` Python hands them, + via `autoprover_sdk::run_confined`. Only file *contents* may derive from the LLM. +* This shape fits a backend whose checker is a **local tool**. One whose "validate" + is a remote service has no host effect to call — see + [docs/rust-applications.md §12](../docs/rust-applications.md). diff --git a/rust/autoprover-sdk/Cargo.toml b/rust/autoprover-sdk/Cargo.toml new file mode 100644 index 00000000..77ecb8aa --- /dev/null +++ b/rust/autoprover-sdk/Cargo.toml @@ -0,0 +1,34 @@ +[package] +name = "autoprover-sdk" +edition.workspace = true +version.workspace = true +license.workspace = true +description = "SDK for building AutoProver formalization backends / applications in Rust, callable from the Python pipeline via PyO3." + +[lib] +name = "autoprover_sdk" + +# The generator half of the wire round-trip fuzzer (`tests/test_wire_roundtrip.py`), which +# drives it over a pipe. Behind `fuzz` so neither `arbitrary` nor this binary enters a shipped +# wheel. +[[bin]] +name = "wire-echo" +path = "src/bin/wire_echo.rs" +required-features = ["fuzz"] + +[features] +fuzz = ["dep:arbitrary"] + +[dependencies] +serde = { workspace = true } +serde_json = { workspace = true } +arbitrary = { version = "1", features = ["derive"], optional = true } +# Re-exported (see `pub use pyo3`) so the `export_app!` macro can reference +# `$crate::pyo3::…` and the app crate needs no direct dependency edge for the +# macro body. The app crate still lists pyo3 to enable `extension-module` / +# `abi3-py312`; cargo feature-unifies the two. +pyo3 = { workspace = true } + +[target.'cfg(unix)'.dependencies] +# process-group SIGKILL on timeout (`kill(-pgid, SIGKILL)`); std has no killpg. +libc = "0.2" diff --git a/rust/autoprover-sdk/src/args.rs b/rust/autoprover-sdk/src/args.rs new file mode 100644 index 00000000..4fb9f77f --- /dev/null +++ b/rust/autoprover-sdk/src/args.rs @@ -0,0 +1,67 @@ +//! The run's inputs as the entry point resolved them — what the two argument-shaped callouts +//! ([`Backend::validate_preconditions`](crate::Backend::validate_preconditions) and +//! [`Backend::sandbox_grants`](crate::Backend::sandbox_grants)) receive. + +use serde::{Deserialize, Serialize}; +use std::path::PathBuf; + +use crate::chain::ChainData; + +/// The parsed values of the descriptor's declared CLI flags, keyed the way the host's argparse +/// keys them: leading dashes stripped and `-` folded to `_` (`--fuzz-timeout` → `fuzz_timeout`). +/// +/// Untyped because the *wheel* declares these flags ([`ArgSpec`](crate::descriptor::ArgSpec)) — +/// the host parses and forwards them without a schema of its own. Read one through +/// [`DeclaredArgs::get`] rather than reaching into the map. +#[derive(Debug, Clone, Default, Serialize, Deserialize)] +#[serde(transparent)] +pub struct DeclaredArgs(serde_json::Map); + +impl DeclaredArgs { + /// One declared flag's value as a `T`. `None` when the flag wasn't declared, was left at a + /// null default, or doesn't hold a `T` — all three being "the operator gave me nothing", which + /// is what a caller's `unwrap_or` handles. + pub fn get(&self, dest: &str) -> Option { + serde_json::from_value(self.0.get(dest)?.clone()).ok() + } + + /// A non-empty string flag. `--flag ""` and an absent flag are the same answer here, which is + /// the check a caller reaching for [`DeclaredArgs::get`] on a path/name flag actually wants. + pub fn text(&self, dest: &str) -> Option { + self.get::(dest).filter(|s| !s.is_empty()) + } +} + +impl From> for DeclaredArgs { + fn from(map: serde_json::Map) -> Self { + DeclaredArgs(map) + } +} + +/// The run's resolved inputs. Every part the host already knows is a field: a callout never has to +/// re-derive one by splitting a joined string (`program` and `source_path` are the two halves of +/// the entry point's `path:Name` argument, split here rather than there). +#[derive(Debug, Clone, Default, Serialize, Deserialize)] +#[cfg_attr(feature = "fuzz", derive(arbitrary::Arbitrary))] +#[serde(deny_unknown_fields)] +pub struct AppArgs { + /// The project root, absolute. + pub project_root: PathBuf, + /// The analysis identifier of the program/contract under test — a label and a namespace, never + /// the name of a build-system unit: see [`AppArgs::source_unit`]. + pub program: String, + /// The main source file, project-root-relative. + pub source_path: String, + /// The design doc, when one was named on the command line. + #[serde(deserialize_with = "crate::required::present")] + pub system_doc: Option, + /// Where the analyzed code lives as a unit of its own build system — + /// [chain-shaped](ChainData), and the same value every later callout receives as + /// [`AuthorInput::source_unit`](crate::authoring::AuthorInput::source_unit). Empty when nothing + /// was resolved. + /// + /// These two callouts run before workspace prep, so there are no prep facts to accompany it. + pub source_unit: ChainData, + /// The wheel's own declared flags. + pub declared: DeclaredArgs, +} diff --git a/rust/autoprover-sdk/src/authoring.rs b/rust/autoprover-sdk/src/authoring.rs new file mode 100644 index 00000000..8a3e3e03 --- /dev/null +++ b/rust/autoprover-sdk/src/authoring.rs @@ -0,0 +1,195 @@ +//! What the host sends *into* the authoring/gating callouts, and the prompt it gets back. + +use serde::{Deserialize, Serialize}; + +use crate::args::DeclaredArgs; +use crate::chain::ChainData; + +/// What kind of thing a property states (mirrors `composer.spec.types.PropertyType`). An enum +/// rather than a free string for the reason [`Outcome`](crate::outcome::Outcome) is one: the set +/// is closed and shared with the host, so a typo fails to compile instead of reaching a prompt. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +#[cfg_attr(feature = "fuzz", derive(arbitrary::Arbitrary))] +pub enum PropertyKind { + AttackVector, + SafetyProperty, + Invariant, +} + +impl PropertyKind { + /// The wire spelling, which is also how a prompt listing properties refers to one. + pub fn as_str(self) -> &'static str { + match self { + Self::AttackVector => "attack_vector", + Self::SafetyProperty => "safety_property", + Self::Invariant => "invariant", + } + } +} + +impl std::fmt::Display for PropertyKind { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(self.as_str()) + } +} + +/// One property to formalize (mirrors `composer.spec.types.PropertyFormulation`), the unit it was +/// inferred for, and the `slug` the host assigned it: unique within the batch, and what a backend +/// names this property's [`Check`](crate::outcome::Check) after (Crucible: `c_`; the example +/// app: `rule_`). +/// +/// The slug is only guaranteed *filesystem*-safe, which is weaker than identifier-safe — a backend +/// that spells it into generated code folds it first. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[cfg_attr(feature = "fuzz", derive(arbitrary::Arbitrary))] +#[serde(deny_unknown_fields)] +pub struct Property { + /// The unit whose analysis produced this property (the host's `FeatureUnit::display_name`). + /// + /// A title identifies a property only within its own unit, so the two together are what + /// identifies one across a run. On a component turn every property names that turn's unit; on a + /// [setup](Authored::Setup) turn, which is sent every unit's properties at once, this is how two + /// same-titled properties are told apart and how each is tied to the surface it must be + /// checkable against. + pub component: String, + pub title: String, + pub sort: PropertyKind, + pub description: String, + pub slug: String, +} + +/// What is being authored or gated, and the payload that exists only for it. The host sends three, +/// and each carries something the other two have nothing to say about — which is why this is a +/// variant rather than a tag beside fields that are empty two times in three. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(tag = "kind", rename_all = "snake_case")] +#[serde(deny_unknown_fields)] +pub enum Authored { + /// Nothing is authored: the wheel renders its own skeleton and `compile` gates the prepared + /// workspace (see [`PhaseRole::Preflight`](crate::descriptor::PhaseRole::Preflight)). It runs + /// before analysis has finished, so there is no model, no unit, and [`AuthorInput::props`] is + /// empty. + Preflight, + /// The shared spec every unit builds on (Crucible's fixture), authored once from the + /// analyzed model and *every* unit's properties (see + /// [`PhaseRole::Setup`](crate::descriptor::PhaseRole::Setup)). + Setup { + /// The analyzed system model, opaque to the SDK — its shape is the ecosystem's. + model: serde_json::Value, + /// Every unit the run is about to formalize, each the same [chain-shaped](ChainData) value a + /// component turn gets as [`Authored::Component::unit`]. + /// + /// This is the only callout that sees the set whole: a per-unit turn holds one, and a + /// preflight runs before any exists. A wheel whose setup gate builds scaffolding the *whole* + /// set implies — a manifest's feature list, a crate root's module declarations — can + /// therefore build the real thing here rather than a provisional form something later has to + /// complete. + units: Vec, + }, + /// One unit's spec. + Component { + /// The unit being formalized, opaque to the SDK — its shape is the ecosystem's + /// (`FeatureUnit::feature_json`). + unit: serde_json::Value, + }, +} + +/// The input to the authoring/gating callouts for one spec. +/// +/// The one wire type without `#[serde(deny_unknown_fields)]`: serde rejects that attribute +/// alongside a `flatten` field, and [`Authored`] has to flatten so the tag and the payload it +/// selects arrive at the same level as everything else. A field the host declares and this side +/// doesn't is therefore silently ignored *here* — caught by `tests/test_wire_roundtrip.py`, which +/// re-reads what this side made of a payload, rather than at the callout. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[cfg_attr(feature = "fuzz", derive(arbitrary::Arbitrary))] +pub struct AuthorInput { + /// What is being authored, with its payload. + #[serde(flatten)] + pub authored: Authored, + /// The analysis identifier of the program/contract under test (the `Name` half of the host's + /// `path:Name` argument) — a label and a namespace, never the name of a build-system unit: that + /// is [`AuthorInput::source_unit`], and the two are independent. + pub program: String, + /// Where the analyzed code lives as a unit of *its own* build system, resolved once per run by + /// the chain's registered `ProjectToolchain` and carried unchanged from here on — so prep, every + /// gated build and the delivered artifact all name the same thing. + /// + /// [Chain-shaped](ChainData): a Cargo crate is a directory, a package name, a lib target and an + /// Anchor requirement; a Move package is not. Empty when nothing was resolved, which is when a + /// wheel applies its own convention. + pub source_unit: ChainData, + /// The properties this spec must make checkable. Empty for a preflight; for a setup, every + /// unit's. + pub props: Vec, + /// The compiled shared setup spec, for a wheel that declared a + /// [`PhaseRole::Setup`](crate::descriptor::PhaseRole::Setup) phase — the fixture a component's + /// spec builds on. + #[serde(deserialize_with = "crate::required::present")] + pub setup: Option, + /// What workspace prep established, from the wheel's own + /// [`WorkspacePrep::toolchain_request`](crate::prep::WorkspacePrep::toolchain_request) — + /// [chain-shaped](ChainData) like [`AuthorInput::source_unit`], and produced by the same + /// `ProjectToolchain`. Empty when the plan asked for nothing beyond placing files. + /// + /// A fact here means *the thing it describes is in place*, which is what a wheel reads to decide + /// how it sources the program's types (Solana: a generated IDL, or a dependency on the crate). + pub prep_facts: ChainData, + /// The run's values for the wheel's own declared flags. + pub args: DeclaredArgs, +} + +impl AuthorInput { + /// The unit being formalized, on a component turn. `None` on the two turns that formalize no + /// unit — a preflight (nothing is analyzed yet) and the shared setup spec. + pub fn unit(&self) -> Option<&serde_json::Value> { + match &self.authored { + Authored::Component { unit } => Some(unit), + _ => None, + } + } + + /// The analyzed system model, on a setup turn. + pub fn model(&self) -> Option<&serde_json::Value> { + match &self.authored { + Authored::Setup { model, .. } => Some(model), + _ => None, + } + } + + /// Every unit the run is about to formalize, on a setup turn. Empty on the turns that hold no + /// set: a preflight (nothing is analyzed yet), and a component turn, which holds its own + /// [`unit`](Self::unit) instead. + pub fn units(&self) -> &[ChainData] { + match &self.authored { + Authored::Setup { units, .. } => units, + _ => &[], + } + } +} + +/// An authoring instruction (+ optional backend-defined system prompt) for one LLM turn. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[cfg_attr(feature = "fuzz", derive(arbitrary::Arbitrary))] +#[serde(deny_unknown_fields)] +pub struct Prompt { + #[serde(deserialize_with = "crate::required::present")] + pub system: Option, + pub instruction: String, +} + +/// The reviewer a wheel declares for an input — everything about a review that is fixed for the +/// whole authoring session, which is why [`Backend::judge`](crate::Backend::judge) is asked once +/// and without a draft. What to ask about a *particular* draft is +/// [`Backend::judge_instruction`](crate::Backend::judge_instruction), per round. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[cfg_attr(feature = "fuzz", derive(arbitrary::Arbitrary))] +#[serde(deny_unknown_fields)] +pub struct Judge { + /// The *domain* half of the reviewer's system prompt — who is reviewing and what they know. + /// The host appends the review protocol (how the verdict comes back) and falls back to a + /// neutral role when this is `None`. + #[serde(deserialize_with = "crate::required::present")] + pub system: Option, +} diff --git a/rust/autoprover-sdk/src/backend.rs b/rust/autoprover-sdk/src/backend.rs new file mode 100644 index 00000000..d310b503 --- /dev/null +++ b/rust/autoprover-sdk/src/backend.rs @@ -0,0 +1,141 @@ +//! The trait an application implements. + +use std::collections::BTreeMap; + +use crate::args::AppArgs; +use crate::authoring::{AuthorInput, Judge, Prompt}; +use crate::descriptor::AppDescriptor; +use crate::finalize::FinalizeInput; +use crate::outcome::{CompileResult, Target, ValidateOutcome}; +use crate::prep::{SandboxGrants, WorkspacePrep}; +use crate::sandbox::Workspace; + +/// A Rust AutoProver backend — a **passive service** the Python pipeline drives. One instance +/// per wheel; construct it in [`export_app!`](crate::export_app). Metadata/authoring callouts are +/// pure; `compile` and `validate` run the toolchain (via [`Workspace::run`]) and BLOCK — the host +/// calls them off the event loop (`asyncio.to_thread`) while the wheel releases the GIL. +pub trait Backend: Send + Sync + 'static { + /// The declaration the Python host reads at load time. + fn descriptor(&self) -> AppDescriptor; + + /// Validate application-specific preconditions before any service opens. `Err(msg)` aborts. + fn validate_preconditions(&self, _args: &AppArgs) -> Result<(), String> { + Ok(()) + } + + /// Which invocation of the checker a declared check runs under — `None` (the default) makes it + /// its own. Pure; asked once per check name the author declared. + /// + /// This is the whole of what a wheel says about the check set: the *names* are the author's + /// (they are names of things in the artifact, so only the author can know them), while how they + /// are grouped into runs is a backend convention. Crucible answers with the component's harness + /// fn — one campaign for the whole property set; a CVL wheel would answer with the component's + /// conf. + fn target_for(&self, _input: &AuthorInput, _check: &str) -> Option { + None + } + + /// The instruction (+ optional system prompt) to author `input.kind`'s spec, covering all its + /// units. + /// + /// Asked once per authoring session, not once per attempt: the host runs a stateful agent that + /// keeps its buffer and its history across revisions, so build errors and review feedback reach + /// the author as tool results rather than as a re-rendered prompt. `system` is the *domain* + /// half of the system prompt — the host prepends the session protocol (the tools, the publish + /// gate, the citation rules) so no wheel restates it. + fn author_prompt(&self, input: &AuthorInput) -> Prompt; + + /// Reject a spec at write time, cheaply and purely — `Some(complaint)` refuses the write and + /// the buffer keeps its previous contents. Default: accept anything, and let `validate` be the + /// only judge. + /// + /// This is for what can be decided without a toolchain (a parser, a required declaration). It + /// is not a build: it runs on every put and edit, so it must be fast, and it must not spawn + /// anything. + fn check_syntax(&self, _input: &AuthorInput, _spec: &str) -> Option { + None + } + + /// Who reviews this input's drafts, before validation — `None` (the default) skips judging, and + /// the compiler + checker are the judges. + /// + /// Asked once, when the authoring session is built: whether this kind of input is reviewed at + /// all and who reviews it are both fixed for the session, so neither is given a draft — none + /// exists yet. A wheel can answer per kind (review components, not the shared setup spec). + fn judge(&self, _input: &AuthorInput) -> Option { + None + } + + /// What to ask the reviewer about this draft — asked once per review round, and only for an + /// input [`Backend::judge`] claimed. The default asks for a plain review, which is what a wheel + /// with nothing input-specific to say wants. + fn judge_instruction(&self, _input: &AuthorInput, _spec: &str) -> String { + "Review the proposed specification and decide whether it is acceptable as it stands.".into() + } + + /// Compile/typecheck the whole spec once (every check in it shares one build). BLOCKING. + /// + /// Also the preflight gate: for [`Authored::Preflight`](crate::authoring::Authored::Preflight) + /// the `spec` is `None` — nothing is authored, and the wheel supplies its own skeleton — so one + /// implementation covers "does the authored artifact build" and "could *any* artifact build + /// here" (see [`PhaseRole::Preflight`](crate::descriptor::PhaseRole::Preflight)). A wheel whose + /// toolchain would take an empty spec file for a real one can tell the two apart. + fn compile(&self, input: &AuthorInput, spec: Option<&str>, ws: &Workspace) -> CompileResult; + + /// Build + check ONE target against the spec (the fused build gate — no separate compile for + /// components). Returns [`ValidateOutcome::BuildFailed`] to trigger a revision of the whole + /// spec (the build is shared across targets), or a [`Verdict`](crate::outcome::Verdict) for + /// each check the target covers — [`Target::checks`], which the host already grouped. + /// Per-target so the host owns scheduling. BLOCKING. + /// + /// # The verdict contract + /// + /// The check names come from the author, so this is also where a name is held to the artifact. + /// A backend must not answer [`Outcome::Good`](crate::outcome::Outcome::Good) for a check it + /// found no evidence ran: + /// + /// | Situation | Verdict | + /// |---|---| + /// | The checker has no such check | `Error`, with a detail naming it | + /// | It exists but the run never exercised it | `Unknown`, with a detail saying so | + /// | It ran and held | `Good` | + /// | It ran and was refuted | `Bad` + the counterexample | + /// + /// Both non-`Good` cases block the publish gate, which is the point: a declared check with + /// nothing behind it must not stamp a property as verified. What corroborates a check is the + /// backend's own business — a per-rule result from the Prover, a runtime tally of which tagged + /// assertions a campaign evaluated — but *some* evidence is required, and a source-text scan is + /// not it (a tag in a comment or in dead code reads exactly like a check). + /// + /// The residue no mechanism reaches — whether a check that demonstrably ran genuinely verifies + /// the property it claims — is the judge's ([`Backend::judge`]). + fn validate( + &self, + input: &AuthorInput, + spec: &str, + target: &Target, + ws: &Workspace, + ) -> ValidateOutcome; + + /// Extra sandbox grants to union into the host's policy (see [`SandboxGrants`]). Pure; called + /// once before any confined step. Default: no extra grants. + fn sandbox_grants(&self, _args: &AppArgs) -> SandboxGrants { + SandboxGrants::default() + } + + /// Declare the pre-formalization workspace prep (see [`WorkspacePrep`]). Pure — the host + /// executes the returned plan with the shared warm/build helpers, so the network posture + /// stays Python-owned. Default: nothing to prepare. + fn workspace_prep(&self, _input: &AuthorInput) -> WorkspacePrep { + WorkspacePrep::default() + } + + /// Optional run-level artifacts from the full outcome set, as `{relpath: contents}`. + /// + /// Under [`DeliverableMode::Callout`](crate::descriptor::DeliverableMode::Callout) this renders + /// the whole source deliverable (Crucible's one crate) — which is why the outcome set carries + /// each component's authored spec and targets alongside the setup spec and the crate. + fn finalize(&self, _outcomes: &FinalizeInput) -> BTreeMap { + BTreeMap::new() + } +} diff --git a/rust/autoprover-sdk/src/bin/wire_echo.rs b/rust/autoprover-sdk/src/bin/wire_echo.rs new file mode 100644 index 00000000..3c8c9003 --- /dev/null +++ b/rust/autoprover-sdk/src/bin/wire_echo.rs @@ -0,0 +1,196 @@ +//! The Rust half of the wire round-trip fuzzer (`tests/test_wire_roundtrip.py`). +//! +//! Reads and writes one JSON object per line on stdin/stdout. Python keeps this process alive +//! for a whole test instead of starting a new one per example. +//! +//! Two operations, one for each direction: +//! +//! * `echo` — parse a host-built payload as the matching Rust type and write it back. A field +//! this side does not know is dropped, so when the host parses the reply it no longer matches +//! what it sent. That is the outbound check. +//! * `gen` — build a value from bytes the host sends (Hypothesis entropy, not a local RNG) and +//! serialize it. One generator then drives both directions and shrinking still works. The host +//! parses the result, dumps it, and sends it back through `echo` to compare. That is the +//! inbound check. +//! +//! `gen` builds from the Rust type so a field only this side declares still appears. A generator +//! built from the host's schema would never produce that field, and would miss that kind of drift. + +use std::io::{self, BufRead, Write}; + +use arbitrary::{Arbitrary, Unstructured}; +use serde::{Deserialize, Serialize}; +use serde_json::Value; + +use autoprover_sdk::args::AppArgs; +use autoprover_sdk::authoring::{AuthorInput, Judge, Prompt, Property}; +use autoprover_sdk::descriptor::AppDescriptor; +use autoprover_sdk::ffi::CalloutError; +use autoprover_sdk::finalize::{ComponentOutcome, FinalizeComponent, FinalizeInput}; +use autoprover_sdk::outcome::{Check, CompileResult, SkippedProperty, Target, ValidateOutcome}; +use autoprover_sdk::prep::{SandboxGrants, WorkspacePrep}; +use autoprover_sdk::sandbox::Sandbox; + +/// Every payload root that crosses the FFI as a whole message, named the way the host names it. +/// +/// An enum rather than a bare string match: adding a root without teaching [`dispatch`] about it +/// is then a compile error, and a name the host misspells fails as a `Request` that won't +/// deserialize — reported against the field instead of silently falling through. +#[derive(Debug, Clone, Copy, Deserialize)] +#[serde(rename_all = "snake_case")] +enum WireType { + // Outbound — the host sends these, so this side only ever deserializes them. + AppArgs, + AuthorInput, + Target, + FinalizeInput, + /// The confinement wrapper, mirrored by a `TypedDict` rather than a pydantic model + /// (`composer.sandbox.config.BackendSpec`) — a mirror all the same. + Sandbox, + // Inbound — a wheel returns these, so this side is what produces them. + AppDescriptor, + CompileResult, + ValidateOutcome, + Prompt, + Judge, + Checks, + WorkspacePrep, + SandboxGrants, + // The error envelope any inbound callout may return instead of its payload. + CalloutError, + // Nested — never a whole message. Addressable anyway so the field-set check can generate one on + // its own and compare its keys against the host's mirror, rather than having to find them at + // some depth inside a parent, below opaque payloads whose keys are not field names at all. + Property, + SkippedProperty, + FinalizeComponent, + ComponentOutcome, +} + +/// What both operations need of a payload type: the host's two directions, plus generation. +trait Wire: serde::de::DeserializeOwned + Serialize + for<'a> Arbitrary<'a> {} +impl Wire for T where T: serde::de::DeserializeOwned + Serialize + for<'a> Arbitrary<'a> {} + +/// One operation, applied to whichever type [`dispatch`] resolved. A trait rather than a closure +/// because the callee is generic over the payload type and closures cannot be. +trait Op { + fn run(self) -> Result; +} + +/// Resolve `ty` to its Rust type and hand it to `op` — the single place the wire names and the +/// types meet, so neither operation can cover a different set than the other. +fn dispatch(ty: WireType, op: O) -> Result { + match ty { + WireType::AppArgs => op.run::(), + WireType::AuthorInput => op.run::(), + WireType::Target => op.run::(), + WireType::FinalizeInput => op.run::(), + WireType::Sandbox => op.run::(), + WireType::AppDescriptor => op.run::(), + WireType::CompileResult => op.run::(), + WireType::ValidateOutcome => op.run::(), + WireType::Prompt => op.run::(), + WireType::Judge => op.run::(), + WireType::Checks => op.run::>(), + WireType::WorkspacePrep => op.run::(), + WireType::SandboxGrants => op.run::(), + WireType::CalloutError => op.run::(), + WireType::Property => op.run::(), + WireType::SkippedProperty => op.run::(), + WireType::FinalizeComponent => op.run::(), + WireType::ComponentOutcome => op.run::(), + } +} + +struct Echo(Value); + +impl Op for Echo { + fn run(self) -> Result { + Ok(serde_json::to_value(serde_json::from_value::(self.0)?)?) + } +} + +struct Gen<'a>(Unstructured<'a>); + +impl<'a> Op for Gen<'a> { + fn run(mut self) -> Result { + Ok(serde_json::to_value(T::arbitrary(&mut self.0)?)?) + } +} + +/// Why an operation produced no payload — kept apart from a successful answer because the two mean +/// different things to the caller: running out of entropy is the harness's own limit and the +/// example is skipped, while a serde failure is the divergence the fuzzer is looking for. +enum Fault { + Exhausted, + Failed(String), +} + +impl From for Fault { + fn from(e: serde_json::Error) -> Self { + Fault::Failed(e.to_string()) + } +} + +impl From for Fault { + fn from(e: arbitrary::Error) -> Self { + match e { + arbitrary::Error::NotEnoughData => Fault::Exhausted, + other => Fault::Failed(other.to_string()), + } + } +} + +#[derive(Debug, Deserialize)] +#[serde(tag = "op", rename_all = "snake_case")] +enum Request { + Echo { ty: WireType, payload: Value }, + Gen { ty: WireType, entropy: Vec }, +} + +#[derive(Debug, Serialize)] +#[serde(tag = "status", rename_all = "snake_case")] +enum Response { + Ok { payload: Value }, + Exhausted, + Error { message: String }, +} + +impl From> for Response { + fn from(result: Result) -> Self { + match result { + Ok(payload) => Response::Ok { payload }, + Err(Fault::Exhausted) => Response::Exhausted, + Err(Fault::Failed(message)) => Response::Error { message }, + } + } +} + +fn handle(line: &str) -> Response { + match serde_json::from_str::(line) { + Ok(Request::Echo { ty, payload }) => dispatch(ty, Echo(payload)).into(), + Ok(Request::Gen { ty, entropy }) => dispatch(ty, Gen(Unstructured::new(&entropy))).into(), + Err(e) => Response::Error { + message: format!("malformed request: {e}"), + }, + } +} + +fn main() -> io::Result<()> { + let stdin = io::stdin().lock(); + let mut stdout = io::stdout().lock(); + for line in stdin.lines() { + let line = line?; + if line.trim().is_empty() { + continue; + } + // A response that won't serialize would desynchronize the pipe, so it can't be a `?`: + // every request gets exactly one line back. + let response = handle(&line); + let encoded = serde_json::to_string(&response) + .unwrap_or_else(|e| format!(r#"{{"status":"error","message":"unencodable: {e}"}}"#)); + writeln!(stdout, "{encoded}")?; + stdout.flush()?; + } + Ok(()) +} diff --git a/rust/autoprover-sdk/src/chain.rs b/rust/autoprover-sdk/src/chain.rs new file mode 100644 index 00000000..32f22927 --- /dev/null +++ b/rust/autoprover-sdk/src/chain.rs @@ -0,0 +1,113 @@ +//! The one shape on this seam the SDK deliberately does not define: a JSON object whose fields +//! belong to the **chain under analysis** rather than to the framework. +//! +//! An application is written in Rust; the project it analyzes need not be. A Cargo crate's package +//! and lib-target names, an Anchor IDL, a Move package's named addresses are all vocabulary of one +//! ecosystem's build system — knowledge the host has no business holding a shape for, since holding +//! one is what makes the next ecosystem an edit to the framework instead of a registration. +//! +//! So the facts travel opaquely. One chain-scoped implementation produces them (`ProjectToolchain`, +//! in `composer/rustapp/toolchain.py`) and the wheels targeting that chain read them, both through +//! the types a chain's support crate defines. This seam guarantees only that the object arrives +//! whole — and *which* type is inside follows from the descriptor's declared +//! [`ecosystem`](crate::descriptor::AppDescriptor::ecosystem), not from inspecting the keys. +//! +//! It is the same treatment [`Authored`](crate::authoring::Authored)'s `model` and `unit` payloads +//! already get, for the same reason: the analyzed system's shape is the ecosystem's. + +use serde::de::DeserializeOwned; +use serde::{Deserialize, Serialize}; +use serde_json::{Map, Value}; + +/// A JSON object this seam carries but does not interpret — see the module docs. +/// +/// `#[serde(transparent)]`, so the wire form is the bare object and the host mirrors it as a plain +/// `dict[str, Any]`; an empty one is how the host says it established nothing. A non-object fails to +/// deserialize, which keeps "opaque" from also meaning "any JSON at all". +#[derive(Debug, Clone, Default, Serialize, Deserialize)] +#[serde(transparent)] +pub struct ChainData(Map); + +impl ChainData { + /// These facts as the type that owns their shape. + /// + /// A `Result` rather than an `Option`: an empty object and one holding another chain's fields are + /// different problems — the first is the documented "nothing was resolved" (test it with + /// [`ChainData::is_empty`]), while the second means a wheel and a toolchain disagree about the + /// chain, which should be reported rather than read as "nothing". A caller that wants its own + /// convention for both spells that at the call site (`.parse().unwrap_or_default()`). + pub fn parse(&self) -> Result { + T::deserialize(&self.0) + } + + /// Nothing was established — no resolver for this chain, no such unit in the language, a layout + /// that couldn't be read, or a prep that asked for nothing. + pub fn is_empty(&self) -> bool { + self.0.is_empty() + } + + /// The wire form of a value whose shape the chain owns — how a wheel builds the + /// [`WorkspacePrep::toolchain_request`](crate::prep::WorkspacePrep::toolchain_request) it wants + /// executed, from the request type it shares with that chain's toolchain. + pub fn of(value: &T) -> Result { + match serde_json::to_value(value)? { + Value::Object(map) => Ok(ChainData(map)), + other => Err(serde::ser::Error::custom(format!( + "chain data must serialize to a JSON object, got {other}" + ))), + } + } +} + +impl From> for ChainData { + fn from(map: Map) -> Self { + ChainData(map) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + /// Stands in for what a chain's support crate defines — the SDK's tests can't name a real one, + /// which is the property under test. + #[derive(Debug, Default, PartialEq, Serialize, Deserialize)] + struct Facts { + dir: String, + package: String, + } + + #[test] + fn a_chain_shaped_value_round_trips_through_the_opaque_form() { + let facts = Facts { + dir: "programs/lend".into(), + package: "example-lending".into(), + }; + let data = ChainData::of(&facts).expect("an object"); + assert!(!data.is_empty()); + assert_eq!(data.parse::().expect("the same shape"), facts); + } + + #[test] + fn nothing_resolved_and_the_wrong_chains_facts_are_different_answers() { + // Both fail to parse, and a wheel has to be able to tell them apart: the first is the + // documented state a resolver-less chain produces, the second is a wiring bug. + let nothing = ChainData::default(); + assert!(nothing.is_empty() && nothing.parse::().is_err()); + + let other_chain: ChainData = + serde_json::from_str(r#"{"named_addresses":{"vault":"0x1"}}"#).expect("an object"); + assert!(!other_chain.is_empty() && other_chain.parse::().is_err()); + } + + #[test] + fn only_an_object_is_chain_data() { + // "Opaque" is about the *fields*, not the JSON type: a list or a string here would mean the + // two ends disagree about more than a field name. + assert!(serde_json::from_str::("[]").is_err()); + assert!(serde_json::from_str::(r#""idl.json""#).is_err()); + assert!(serde_json::from_str::("{}") + .expect("an object") + .is_empty()); + } +} diff --git a/rust/autoprover-sdk/src/descriptor.rs b/rust/autoprover-sdk/src/descriptor.rs new file mode 100644 index 00000000..99e84c80 --- /dev/null +++ b/rust/autoprover-sdk/src/descriptor.rs @@ -0,0 +1,272 @@ +//! The declarative spine the Python host consumes to synthesize the phase enum, +//! argparse, frontend and artifact store (see `docs/rust-applications.md` §3). + +use serde::{Deserialize, Serialize}; + +/// Which step of the run a declared phase groups — and, for the steps the host runs as their own +/// visible task, the declaration *of* that step: its task is the phase's label under the phase +/// itself. A role no phase claims is a step this application doesn't have. +/// +/// The four the driver itself runs must each be claimed exactly once; the rest are optional. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +#[cfg_attr(feature = "fuzz", derive(arbitrary::Arbitrary))] +pub enum PhaseRole { + /// Grouping only — the host runs no step of its own here (cf. autoprove's harness/autosetup). + #[default] + Grouping, + Analysis, + Extraction, + Formalization, + Report, + /// Design-doc discovery, which the host's entry point runs *before* the pipeline (and only when + /// the doc wasn't given on the command line). Optional: claim it to give that task a section of + /// its own, or leave it and the host groups the task under the first declared phase. + Discovery, + /// The analysis-independent gate on the prepared workspace, run concurrently with system + /// analysis — before a single property exists. The host follows + /// [`WorkspacePrep`](crate::prep::WorkspacePrep) with an + /// [`Authored::Preflight`](crate::authoring::Authored::Preflight) `compile` whose `spec` is + /// **empty**: nothing has been authored yet, so the wheel renders its own skeleton — the smallest + /// spec that still exercises everything an authored one will depend on. + /// + /// The point is to fail on a *toolchain* problem — an unresolvable dependency graph, a harness + /// that doesn't link, IDL codegen the generator rejects — while the run has spent almost no LLM + /// budget. Without it, such a problem first surfaces as compiler errors in the first authored + /// draft, which the author cannot fix: it does not own the manifest, and it burns every + /// re-author attempt trying. So a preflight failure is **terminal** — the host raises instead of + /// re-authoring, and the driver cancels the analysis and extraction running alongside it. + Preflight, + /// The shared spec authored once before per-component formalization (Crucible's fixture). + /// The host runs the author→compile loop for an + /// [`Authored::Setup`](crate::authoring::Authored::Setup) input, then hands the compiled spec + /// to every component as + /// [`AuthorInput::setup`](crate::authoring::AuthorInput::setup). + Setup, +} + +impl PhaseRole { + /// The four steps the shared driver runs itself, and therefore tags every run with. Every + /// application must claim each of them. + pub const REQUIRED: [PhaseRole; 4] = [ + Self::Analysis, + Self::Extraction, + Self::Formalization, + Self::Report, + ]; +} + +/// One task-grouping phase. `key` becomes the synthesized `enum.Enum` member name; `label`/`order` +/// drive UI grouping; `role` says which step of the run it groups (and declares that step). +#[derive(Debug, Clone, Serialize, Deserialize)] +#[cfg_attr(feature = "fuzz", derive(arbitrary::Arbitrary))] +#[serde(deny_unknown_fields)] +pub struct PhaseSpec { + pub key: String, + pub label: String, + pub order: u32, + pub role: PhaseRole, +} + +impl PhaseSpec { + /// A phase that only groups tasks. + pub fn grouping(key: impl Into, label: impl Into, order: u32) -> Self { + Self { + key: key.into(), + label: label.into(), + order, + role: PhaseRole::Grouping, + } + } + + /// A phase that groups — and declares — the step `role`. + pub fn step( + key: impl Into, + label: impl Into, + order: u32, + role: PhaseRole, + ) -> Self { + Self { + key: key.into(), + label: label.into(), + order, + role, + } + } +} + +/// Default value for a declared CLI argument. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(tag = "kind", rename_all = "snake_case")] +#[cfg_attr(feature = "fuzz", derive(arbitrary::Arbitrary))] +#[serde(deny_unknown_fields)] +pub enum ArgDefault { + Str { value: Option }, + Int { value: Option }, + Bool { value: bool }, +} + +/// A CLI flag the generic entry point adds beyond the three positional inputs +/// (`project_root`, `main_contract`, `system_doc`). +#[derive(Debug, Clone, Serialize, Deserialize)] +#[cfg_attr(feature = "fuzz", derive(arbitrary::Arbitrary))] +#[serde(deny_unknown_fields)] +pub struct ArgSpec { + pub flag: String, + pub help: String, + pub default: ArgDefault, + pub required: bool, +} + +/// A domain event kind the frontend should render. +/// +/// A `notice` kind is surfaced as a persistent, always-visible callout (plus a toast) +/// rather than a line in the collapsible per-task events log — for one-shot important +/// results such as one check's verdict. Ordinary kinds stream into the log. +/// +/// The events themselves are emitted by the **host**, around the callouts it drives (a build +/// failure, a review verdict, a check's outcome): a wheel has no emit channel, because its blocking +/// callouts run to completion with the GIL released. Declaring a kind nothing emits renders +/// nothing. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[cfg_attr(feature = "fuzz", derive(arbitrary::Arbitrary))] +#[serde(deny_unknown_fields)] +pub struct EventKind { + pub kind: String, + pub label: String, + pub notice: bool, +} + +impl EventKind { + /// A streaming event kind — rendered as a line in the collapsible events log. + pub fn log(kind: impl Into, label: impl Into) -> Self { + Self { + kind: kind.into(), + label: label.into(), + notice: false, + } + } + + /// A notice event kind — surfaced as a persistent callout + toast. + pub fn notice(kind: impl Into, label: impl Into) -> Self { + Self { + kind: kind.into(), + label: label.into(), + notice: true, + } + } +} + +/// On-disk deliverable layout. All paths are project-root-relative. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[cfg_attr(feature = "fuzz", derive(arbitrary::Arbitrary))] +#[serde(deny_unknown_fields)] +pub struct ArtifactLayout { + pub deliverable_dir: String, + pub internal_dir: String, + pub report_dir: String, + /// Where the verification artifacts themselves are written. + pub artifact_dir: String, + /// Filename prefix for a per-component artifact (e.g. `autospec` → `autospec_.spec`). + pub artifact_prefix: String, + /// Artifact file extension, no dot (e.g. `spec`, `t.sol`). + pub artifact_extension: String, + /// The store's term for the property→check map file suffix (`property_rules`, `property_tests`). + pub property_suffix: String, +} + +/// How the source deliverable is written to disk. +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "mode", rename_all = "snake_case")] +#[cfg_attr(feature = "fuzz", derive(arbitrary::Arbitrary))] +#[serde(deny_unknown_fields)] +pub enum DeliverableMode { + /// The generic store writes one `{prefix}_{slug}.{ext}` file per component from its + /// `artifact_text`. + #[default] + PerComponent, + /// The store writes no per-component source; the wheel's `finalize` renders the whole + /// deliverable (e.g. Crucible's one shared crate assembled from all sections + the fixture). + Callout { + /// Does the deliverable have a representative primary file? `finalize` renders a whole + /// tree, but every delivered component still records one path — its basename becomes the + /// component's `unit_file`, the report's rule-identity fallback, echoed back to + /// `finalize` — and the store can't guess where in that tree the components' checks land. + /// A path (project-relative, `{program}`-templated — Crucible: + /// `fuzz/{program}/src/main.rs`) names that file; `None` declares that no one file + /// represents the deliverable, and components anchor to the layout's `deliverable_dir` + /// instead. The wire requires the key either way, so opting out is an explicit choice, + /// not an omitted field. Carried by the variant because it means nothing under + /// `PerComponent`. + #[serde(deserialize_with = "crate::required::present")] + deliverable_path: Option, + }, +} + +/// The complete declaration the Python host reads once at load time. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct AppDescriptor { + pub name: String, + pub header_text: String, + /// The ecosystem (chain) tag: "evm" | "solana" | "soroban". Selects the shared front + /// half's system model + prompts; the Python host resolves it against its ecosystem + /// registry, and rejects a tag it doesn't know. + pub ecosystem: String, + /// The report's backend tag (`AutoProverReport.backend`). + pub backend_tag: String, + /// Prose injected into the property-extraction prompt (verification-surface guidance). + pub backend_guidance: String, + /// The system-analysis cache key (`SystemAnalysisSpec.analysis_key`). + pub analysis_key: String, + pub phases: Vec, + pub args: Vec, + #[serde(deserialize_with = "crate::required::present")] + pub rag_db_default: Option, + pub event_kinds: Vec, + pub artifact_layout: ArtifactLayout, + /// How the source deliverable is written (see [`DeliverableMode`]). + pub deliverable_mode: DeliverableMode, + /// Serialize the blocking toolchain callouts (`prepare_workspace`/`compile`/`validate`) on + /// one semaphore — set when the app shares a single build dir / target across components. + pub serialize_toolchain: bool, + /// Default to the fail-closed `launcher` sandbox provider (still overridable by + /// `COMPOSER_SANDBOX_PROVIDER`). Set by any wheel that runs untrusted native toolchains. + pub confine_by_default: bool, + /// Human noun for one formalized component in the console/TUI summary ("instruction" for + /// Crucible). Defaults to "component". + #[serde(deserialize_with = "crate::required::present")] + pub component_noun: Option, + /// What this backend calls one [`Check`](crate::outcome::Check) *to the model* — "rule", + /// "harness function", "invariant". It is the word the authoring prompts use throughout, so it + /// should be the word the wheel's own prompts and its generated code already use. + /// + /// Declared rather than fixed because an author writes better when the prompt speaks its + /// domain's language; the host's tool *names* stay `check`-worded either way, so this changes + /// prose, not the API the model calls. `None` → "check". + #[serde(deserialize_with = "crate::required::present")] + pub check_noun: Option, + /// What an author may cite when it rebuts the judge's prior-round feedback. The host builds the + /// rebuttal tool's `evidence_type` from this, so it is a closed set the model picks from. + /// + /// Declared per wheel because the evidence a backend can actually produce is a property of that + /// backend: a fuzzing wheel can show a counterexample, a typechecking one an error from a + /// checker that never runs code. See [`EVIDENCE_KINDS`] for the default set. + pub evidence_kinds: Vec, +} + +/// The evidence an author can usually offer, for a wheel with no reason to name its own: the build +/// failed, the checker said so, the checker produced a counterexample, the manual says so, or the +/// author is arguing. The last is deliberately last — an argument is a conversation, not a veto. +pub const EVIDENCE_KINDS: [&str; 5] = [ + "build_failure", + "check_output", + "counterexample", + "manual_citation", + "reasoned", +]; + +/// [`EVIDENCE_KINDS`] as the descriptor field wants it. +pub fn default_evidence_kinds() -> Vec { + EVIDENCE_KINDS.iter().map(|s| (*s).to_string()).collect() +} diff --git a/rust/autoprover-sdk/src/export.rs b/rust/autoprover-sdk/src/export.rs new file mode 100644 index 00000000..7d1426b3 --- /dev/null +++ b/rust/autoprover-sdk/src/export.rs @@ -0,0 +1,151 @@ +//! The export macro. `#[macro_export]` puts [`export_app!`](crate::export_app) at the crate root +//! regardless of this module. + +/// Emit the PyO3 module the Python host loads. Invoke it once in an application crate +/// (a `cdylib` depending on `autoprover-sdk` and `pyo3`): +/// +/// ```ignore +/// autoprover_sdk::export_app!(my_app, MyApp::new()); +/// ``` +/// +/// `module_ident` MUST match the wheel's module name. The expansion defines the pure callouts +/// (`descriptor`/`validate_preconditions`/`target_for`/`author_prompt`/`check_syntax`/`judge`/ +/// `judge_instruction`/`finalize`) and +/// the two BLOCKING ones (`compile`/`validate`, which release the GIL while `run-confined` runs), +/// all delegating to the [`ffi`](crate::ffi) helpers of the same name. +#[macro_export] +macro_rules! export_app { + ($module:ident, $ctor:expr) => { + fn __autoprover_app() -> &'static dyn $crate::Backend { + static APP: ::std::sync::OnceLock<::std::boxed::Box> = + ::std::sync::OnceLock::new(); + &**APP.get_or_init(|| ::std::boxed::Box::new($ctor)) + } + + #[$crate::pyo3::pyfunction] + fn descriptor() -> ::std::string::String { + $crate::ffi::descriptor(__autoprover_app()) + } + + #[$crate::pyo3::pyfunction] + fn validate_preconditions( + args_json: ::std::string::String, + ) -> ::std::option::Option<::std::string::String> { + $crate::ffi::validate_preconditions(__autoprover_app(), &args_json) + } + + #[$crate::pyo3::pyfunction] + fn target_for( + input_json: ::std::string::String, + check: ::std::string::String, + ) -> ::std::option::Option<::std::string::String> { + $crate::ffi::target_for(__autoprover_app(), &input_json, &check) + } + + #[$crate::pyo3::pyfunction] + fn author_prompt(input_json: ::std::string::String) -> ::std::string::String { + $crate::ffi::author_prompt(__autoprover_app(), &input_json) + } + + #[$crate::pyo3::pyfunction] + fn check_syntax( + input_json: ::std::string::String, + spec: ::std::string::String, + ) -> ::std::option::Option<::std::string::String> { + $crate::ffi::check_syntax(__autoprover_app(), &input_json, &spec) + } + + #[$crate::pyo3::pyfunction] + fn judge( + input_json: ::std::string::String, + ) -> ::std::option::Option<::std::string::String> { + $crate::ffi::judge(__autoprover_app(), &input_json) + } + + #[$crate::pyo3::pyfunction] + fn judge_instruction( + input_json: ::std::string::String, + spec: ::std::string::String, + ) -> ::std::string::String { + $crate::ffi::judge_instruction(__autoprover_app(), &input_json, &spec) + } + + #[$crate::pyo3::pyfunction] + fn compile( + py: $crate::pyo3::Python<'_>, + input_json: ::std::string::String, + spec: ::std::option::Option<::std::string::String>, + workdir: ::std::string::String, + sandbox_json: ::std::string::String, + ) -> ::std::string::String { + // Release the GIL for the (minutes-long) build — no async runtime needed. + py.allow_threads(move || { + $crate::ffi::compile( + __autoprover_app(), + &input_json, + spec.as_deref(), + &workdir, + &sandbox_json, + ) + }) + } + + #[$crate::pyo3::pyfunction] + fn validate( + py: $crate::pyo3::Python<'_>, + input_json: ::std::string::String, + spec: ::std::string::String, + target_json: ::std::string::String, + workdir: ::std::string::String, + sandbox_json: ::std::string::String, + ) -> ::std::string::String { + py.allow_threads(move || { + $crate::ffi::validate( + __autoprover_app(), + &input_json, + &spec, + &target_json, + &workdir, + &sandbox_json, + ) + }) + } + + #[$crate::pyo3::pyfunction] + fn sandbox_grants(args_json: ::std::string::String) -> ::std::string::String { + $crate::ffi::sandbox_grants(__autoprover_app(), &args_json) + } + + #[$crate::pyo3::pyfunction] + fn workspace_prep(input_json: ::std::string::String) -> ::std::string::String { + $crate::ffi::workspace_prep(__autoprover_app(), &input_json) + } + + #[$crate::pyo3::pyfunction] + fn finalize( + outcomes_json: ::std::string::String, + ) -> ::std::option::Option<::std::string::String> { + $crate::ffi::finalize(__autoprover_app(), &outcomes_json) + } + + #[$crate::pyo3::pymodule] + fn $module( + m: &$crate::pyo3::Bound<'_, $crate::pyo3::types::PyModule>, + ) -> $crate::pyo3::PyResult<()> { + use $crate::pyo3::types::PyModuleMethods as _; + m.add_function($crate::pyo3::wrap_pyfunction!(descriptor, m)?)?; + m.add_function($crate::pyo3::wrap_pyfunction!(validate_preconditions, m)?)?; + m.add_function($crate::pyo3::wrap_pyfunction!(target_for, m)?)?; + m.add_function($crate::pyo3::wrap_pyfunction!(author_prompt, m)?)?; + m.add_function($crate::pyo3::wrap_pyfunction!(check_syntax, m)?)?; + m.add_function($crate::pyo3::wrap_pyfunction!(judge, m)?)?; + m.add_function($crate::pyo3::wrap_pyfunction!(judge_instruction, m)?)?; + m.add_function($crate::pyo3::wrap_pyfunction!(compile, m)?)?; + m.add_function($crate::pyo3::wrap_pyfunction!(validate, m)?)?; + m.add_function($crate::pyo3::wrap_pyfunction!(sandbox_grants, m)?)?; + m.add_function($crate::pyo3::wrap_pyfunction!(workspace_prep, m)?)?; + m.add_function($crate::pyo3::wrap_pyfunction!(finalize, m)?)?; + ::std::result::Result::Ok(()) + } + }; +} diff --git a/rust/autoprover-sdk/src/ffi.rs b/rust/autoprover-sdk/src/ffi.rs new file mode 100644 index 00000000..83d707f7 --- /dev/null +++ b/rust/autoprover-sdk/src/ffi.rs @@ -0,0 +1,483 @@ +//! The sync, JSON-string boundary. [`export_app!`](crate::export_app) wraps these in +//! `#[pyfunction]`s (compile/validate release the GIL); also unit-testable without Python. +//! +//! Every callout keeps a [`Result`] until this module writes the string the host reads. `Ok` is +//! the payload type unchanged. `Err` is always [`CalloutError`] — the one extra inbound JSON +//! shape — so a host bug cannot be read as an empty plan, a skipped review, or a failed build. + +use crate::args::AppArgs; +use crate::authoring::AuthorInput; +use crate::backend::Backend; +use crate::sandbox::Workspace; +use serde::{Deserialize, Serialize}; + +/// Why a callout produced no payload. Success is the payload type unchanged; this is the only +/// extra JSON shape on the inbound side of the seam. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "kind", rename_all = "snake_case")] +#[cfg_attr(feature = "fuzz", derive(arbitrary::Arbitrary))] +#[serde(deny_unknown_fields)] +pub enum CalloutError { + Error { message: String }, +} + +/// JSON for a successful payload, or [`CalloutError`] when the callout could not produce one. +pub fn encode(result: Result) -> String { + match result { + Ok(value) => match serde_json::to_string(&value) { + Ok(json) => json, + Err(e) => encode_err(e), + }, + Err(e) => encode_err(e), + } +} + +fn encode_err(e: impl ToString) -> String { + match serde_json::to_string(&CalloutError::Error { + message: e.to_string(), + }) { + Ok(json) => json, + Err(_) => "{\"kind\":\"error\",\"message\":\"unserializable error\"}".into(), + } +} + +/// `None` is a successful empty answer (no judge, no files). A failed callout is `Some` of the +/// error envelope, never `None`. +fn encode_opt(result: Result, String>) -> Option { + match result { + Ok(None) => None, + Ok(Some(value)) => Some(encode(Ok(value))), + Err(e) => Some(encode_err(e)), + } +} + +/// Like [`encode_opt`], but a successful `Some` is returned as-is — a target name, a precondition +/// or syntax complaint — rather than JSON-encoded. +fn encode_opt_text(result: Result, String>) -> Option { + match result { + Ok(value) => value, + Err(e) => Some(encode_err(e)), + } +} + +fn parse(json: &str, what: &str) -> Result { + serde_json::from_str(json).map_err(|e| format!("invalid {what} JSON: {e}")) +} + +/// Parse an [`AuthorInput`]. +/// +/// Nothing is normalized on the way through: the project facts a payload carries are +/// [chain-shaped](crate::chain::ChainData), so filling in an unresolved one means applying *one +/// ecosystem's* layout convention — which is the wheel's business, or its chain support crate's, not +/// this boundary's. +fn parse_input(json: &str) -> Result { + parse(json, "AuthorInput") +} + +/// The workspace a blocking callout runs in, from the two strings the host sends it as. +fn workspace(workdir: &str, sandbox_json: &str) -> Result { + Ok(Workspace { + dir: std::path::PathBuf::from(workdir), + sandbox: parse(sandbox_json, "Sandbox")?, + }) +} + +fn parse_args(json: &str) -> Result { + parse(json, "AppArgs") +} + +/// `descriptor() -> str` (JSON). +pub fn descriptor(b: &dyn Backend) -> String { + encode(Ok(b.descriptor())) +} + +/// `validate_preconditions(args_json) -> str | None` (None = ok). A payload this can't parse is a +/// [`CalloutError`], not a failed precondition: the host sent the args and a parse failure is a +/// protocol bug, not something the wheel asked to refuse. +pub fn validate_preconditions(b: &dyn Backend, args_json: &str) -> Option { + encode_opt_text(parse_args(args_json).map(|args| b.validate_preconditions(&args).err())) +} + +/// `target_for(input_json, check) -> str | None` — the target the named check runs under, `None` +/// for its own. Pure. An unparseable input is a [`CalloutError`], not `None`: `None` means the +/// check is its own target. +pub fn target_for(b: &dyn Backend, input_json: &str, check: &str) -> Option { + encode_opt_text(parse_input(input_json).map(|input| b.target_for(&input, check))) +} + +/// `author_prompt(input_json) -> str` (JSON `Prompt`). +pub fn author_prompt(b: &dyn Backend, input_json: &str) -> String { + encode(parse_input(input_json).map(|input| b.author_prompt(&input))) +} + +/// `check_syntax(input_json, spec) -> str | None` (None = the spec may be written). A payload this +/// can't parse is a [`CalloutError`]: the alternative is accepting a spec no backend ever saw. +pub fn check_syntax(b: &dyn Backend, input_json: &str, spec: &str) -> Option { + encode_opt_text(parse_input(input_json).map(|input| b.check_syntax(&input, spec))) +} + +/// `judge(input_json) -> str | None` (JSON `Judge`; None = this wheel does not review this input). +/// Takes no spec: it is asked once, before anything is authored. An unparseable input is a +/// [`CalloutError`], not `None`: `None` means no judge. +pub fn judge(b: &dyn Backend, input_json: &str) -> Option { + encode_opt(parse_input(input_json).map(|input| b.judge(&input))) +} + +/// `judge_instruction(input_json, spec) -> str` — the instruction itself, not JSON. Asked per review +/// round, only for an input `judge` claimed. An unparseable payload is a [`CalloutError`] rather +/// than an instruction the reviewer would be asked to follow. +pub fn judge_instruction(b: &dyn Backend, input_json: &str, spec: &str) -> String { + match parse_input(input_json) { + Ok(input) => b.judge_instruction(&input, spec), + Err(e) => encode_err(e), + } +} + +/// `compile(input_json, spec | None, workdir, sandbox_json) -> str` (JSON `CompileResult`). +/// BLOCKING. `None` is the preflight: nothing has been authored, so there is no spec at all. +/// A payload this can't parse is a [`CalloutError`], not a failed build: the author cannot fix +/// a host bug by revising the spec. +pub fn compile( + b: &dyn Backend, + input_json: &str, + spec: Option<&str>, + workdir: &str, + sandbox_json: &str, +) -> String { + encode((|| { + let input = parse_input(input_json)?; + let ws = workspace(workdir, sandbox_json)?; + Ok(b.compile(&input, spec, &ws)) + })()) +} + +/// `validate(input_json, spec, target_json, workdir, sandbox_json) -> str` (JSON +/// `ValidateOutcome`). BLOCKING. +/// +/// An unparseable target (or input, or sandbox) is a [`CalloutError`]. The host already has the +/// target it sent, so a coverage miss would only hide the parse failure behind a different one. +pub fn validate( + b: &dyn Backend, + input_json: &str, + spec: &str, + target_json: &str, + workdir: &str, + sandbox_json: &str, +) -> String { + encode((|| { + let input = parse_input(input_json)?; + let target = parse(target_json, "Target")?; + let ws = workspace(workdir, sandbox_json)?; + Ok(b.validate(&input, spec, &target, &ws)) + })()) +} + +/// `sandbox_grants(args_json) -> str` (JSON `SandboxGrants`). +pub fn sandbox_grants(b: &dyn Backend, args_json: &str) -> String { + encode(parse_args(args_json).map(|args| b.sandbox_grants(&args))) +} + +/// `workspace_prep(input_json) -> str` (JSON `WorkspacePrep`). Pure. +pub fn workspace_prep(b: &dyn Backend, input_json: &str) -> String { + encode(parse_input(input_json).map(|input| b.workspace_prep(&input))) +} + +/// `finalize(outcomes_json) -> str | None` (JSON `{relpath: contents}`, or None). +pub fn finalize(b: &dyn Backend, outcomes_json: &str) -> Option { + encode_opt(parse(outcomes_json, "FinalizeInput").map(|outcomes| { + let files = b.finalize(&outcomes); + (!files.is_empty()).then_some(files) + })) +} + +#[cfg(test)] +mod tests { + //! The boundary's own guarantees: what a backend receives is what the host sent, and a payload's + //! `kind` decides what there is to read. + use super::*; + use crate::authoring::Prompt; + use crate::chain::ChainData; + use crate::descriptor::AppDescriptor; + use crate::finalize::{ComponentOutcome, FinalizeInput}; + use crate::outcome::{CompileResult, Target, ValidateOutcome}; + use crate::prep::WorkspacePrep; + use serde::{Deserialize, Serialize}; + use std::sync::Mutex; + + /// Stands in for the type a chain's support crate defines. The SDK cannot name a real one — that + /// it doesn't have to is the property these tests cover — so the Cargo shape is spelled here, + /// where it is just another wheel's idea of what a project is. + #[derive(Debug, Default, PartialEq, Serialize, Deserialize)] + struct CargoUnit { + dir: String, + package: String, + lib: String, + } + + /// One `AuthorInput` with every field the wire requires, so a test can vary the one it is about. + /// Absence is an error on this seam (see `crate::required`), which is exactly why a fixture that + /// spells only what it cares about would fail to parse. + fn component_json(source_unit: &str, prep_facts: &str) -> String { + format!( + r#"{{"kind":"component","program":"vault","unit":{{"slug":"farms"}}, + "source_unit":{source_unit},"prep_facts":{prep_facts}, + "props":[],"setup":null,"args":{{}}}}"# + ) + } + + /// Records the project facts each callout was handed, so a test can assert on what crossed the + /// seam rather than on what a backend chose to do with them. + #[derive(Default)] + struct Spy { + seen: Mutex>, + } + + impl Backend for Spy { + fn descriptor(&self) -> AppDescriptor { + unimplemented!("not exercised") + } + fn target_for(&self, input: &AuthorInput, _check: &str) -> Option { + self.seen.lock().unwrap().push(input.source_unit.clone()); + None + } + fn author_prompt(&self, _input: &AuthorInput) -> Prompt { + unimplemented!("not exercised") + } + fn compile( + &self, + _input: &AuthorInput, + _spec: Option<&str>, + _ws: &Workspace, + ) -> CompileResult { + unimplemented!("not exercised") + } + fn validate( + &self, + _input: &AuthorInput, + _spec: &str, + _target: &Target, + _ws: &Workspace, + ) -> ValidateOutcome { + unimplemented!("not exercised") + } + fn workspace_prep(&self, input: &AuthorInput) -> WorkspacePrep { + self.seen.lock().unwrap().push(input.source_unit.clone()); + WorkspacePrep::default() + } + fn validate_preconditions(&self, args: &AppArgs) -> Result<(), String> { + self.seen.lock().unwrap().push(args.source_unit.clone()); + Ok(()) + } + } + + #[test] + fn the_projects_own_shape_crosses_the_boundary_untouched() { + // The lend shape: directory, package and lib all differ from the analysis identifier and + // from each other. Every callout carrying project facts gets them verbatim — this boundary + // knows no layout convention to "helpfully" apply, which is what lets a wheel for a chain + // with no crates use the same seam. + let spy = Spy::default(); + let unit = r#"{"dir":"programs/lend","package":"example-lending","lib":"example_lending"}"#; + target_for(&spy, &component_json(unit, "{}"), "r_x"); + workspace_prep(&spy, &component_json(unit, "{}")); + validate_preconditions( + &spy, + &format!( + r#"{{"project_root":"/p","program":"vault","source_path":"src/lib.rs", + "system_doc":null,"source_unit":{unit},"declared":{{}}}}"# + ), + ); + let seen = spy.seen.lock().unwrap(); + assert_eq!( + seen.len(), + 3, + "every callout that carries project facts was exercised" + ); + for data in seen.iter() { + assert_eq!( + data.parse::().expect("the chain's own shape"), + CargoUnit { + dir: "programs/lend".into(), + package: "example-lending".into(), + lib: "example_lending".into(), + } + ); + } + } + + #[test] + fn an_unresolved_project_reaches_the_backend_empty() { + // The host resolved nothing (no toolchain registered for the chain, no such unit in the + // language, an unreadable layout). The wheel is told exactly that, and applies its own + // convention if it has one — the alternative, filling it in here, would mean this seam + // choosing one ecosystem's layout for every wheel. + let spy = Spy::default(); + target_for(&spy, &component_json("{}", "{}"), "r_x"); + let seen = spy.seen.lock().unwrap(); + let data = seen.first().expect("target_for was called"); + assert!(data.is_empty()); + assert!( + data.parse::().is_err(), + "empty is not a resolved unit" + ); + } + + #[test] + fn an_input_carries_only_the_payload_its_kind_has() { + // The host's wire shape is flat: `kind` selects the variant, and the field beside it + // belongs to that variant alone. A component turn has no analyzed model to read, and a + // preflight has neither — it runs before anything is analyzed. + let comp: AuthorInput = serde_json::from_str( + r#"{"kind":"component","program":"vault","unit":{"slug":"farms"}, + "source_unit":{},"prep_facts":{"idl":"fuzz/vault/idls/vault.json"}, + "props":[],"setup":"struct Fixture {}","args":{"fuzz_timeout":900}}"#, + ) + .expect("parse"); + assert_eq!( + comp.unit() + .and_then(|u| u.get("slug")) + .and_then(|v| v.as_str()), + Some("farms") + ); + assert!(comp.model().is_none()); + assert_eq!(comp.setup.as_deref(), Some("struct Fixture {}")); + assert_eq!(comp.args.get::("fuzz_timeout"), Some(900)); + // An absent flag and one left at a null default are the same answer. + assert_eq!(comp.args.get::("nope"), None); + + let setup: AuthorInput = serde_json::from_str( + r#"{"kind":"setup","program":"vault","model":{"components":[]}, + "units":[{"slug":"farms"},{"slug":"vaults"}], + "source_unit":{},"prep_facts":{},"props":[],"setup":null,"args":{}}"#, + ) + .expect("parse"); + assert!(setup.model().is_some() && setup.unit().is_none()); + // The one turn holding the whole unit set: it is the run's, not this spec's, which is why + // it arrives here and not through `unit()`. + assert_eq!(setup.units().len(), 2); + assert!( + comp.units().is_empty(), + "a component turn holds its own unit, not the set" + ); + assert!( + setup.prep_facts.is_empty(), + "a prep that established nothing says so" + ); + + let pre: AuthorInput = serde_json::from_str( + r#"{"kind":"preflight","program":"vault","source_unit":{},"prep_facts":{}, + "props":[],"setup":null,"args":{}}"#, + ) + .expect("parse"); + assert!(pre.unit().is_none() && pre.model().is_none() && pre.props.is_empty()); + assert!( + pre.units().is_empty(), + "nothing is analyzed yet, so there is no unit set" + ); + + // …and it round-trips flat, which is what the host parses back. + let json = serde_json::to_value(&pre).expect("serialize"); + assert_eq!(json.get("kind").and_then(|v| v.as_str()), Some("preflight")); + } + + fn assert_error(raw: &str, needle: &str) { + let CalloutError::Error { message } = serde_json::from_str(raw).expect("error envelope"); + assert!(message.contains(needle), "{message}"); + } + + #[test] + fn a_malformed_payload_is_the_error_envelope() { + // None of these may look like a successful empty answer: no judge, no files, the check is + // its own target, a failed build the author should revise. + let spy = Spy::default(); + let input = component_json("{}", "{}"); + assert_error(&author_prompt(&spy, "not json"), "AuthorInput"); + assert_error( + &compile(&spy, "not json", None, "/tmp", "{}"), + "AuthorInput", + ); + assert_error(&compile(&spy, &input, None, "/tmp", "not json"), "Sandbox"); + let target = r#"{"name":"t","checks":[]}"#; + assert_error( + &validate(&spy, "not json", "", target, "/tmp", "{}"), + "AuthorInput", + ); + assert_error( + &validate(&spy, &input, "", "not json", "/tmp", "{}"), + "Target", + ); + assert_error( + &validate(&spy, &input, "", target, "/tmp", "not json"), + "Sandbox", + ); + assert_error(&workspace_prep(&spy, "not json"), "AuthorInput"); + assert_error(&sandbox_grants(&spy, "not json"), "AppArgs"); + assert_error(&judge_instruction(&spy, "not json", ""), "AuthorInput"); + assert_error( + judge(&spy, "not json").as_deref().expect("Err is Some"), + "AuthorInput", + ); + assert_error( + target_for(&spy, "not json", "c") + .as_deref() + .expect("Err is Some"), + "AuthorInput", + ); + assert_error( + validate_preconditions(&spy, "not json") + .as_deref() + .expect("Err is Some"), + "AppArgs", + ); + assert_error( + check_syntax(&spy, "not json", "") + .as_deref() + .expect("Err is Some"), + "AuthorInput", + ); + assert_error( + finalize(&spy, "not json").as_deref().expect("Err is Some"), + "FinalizeInput", + ); + assert!( + judge(&spy, &input).is_none(), + "a valid input with no judge stays None" + ); + assert!( + target_for(&spy, &input, "c").is_none(), + "Spy groups each check as its own" + ); + } + + #[test] + fn the_outcome_set_parses_the_hosts_payload() { + let input: FinalizeInput = serde_json::from_str( + r#"{"program":"lending", + "source_unit":{"dir":"p","package":"l","lib":"l"},"prep_facts":{}, + "setup":"struct Fixture {}","components":[ + {"name":"Farms","outcome":{"status":"delivered","artifact_text":"fn c_farms(){}", + "targets":["c_farms"],"property_checks":[["fifo",["c_fifo"]]], + "skipped":[],"unit_file":null,"run_link":null}}, + {"name":"Referrals","outcome":{"status":"gave_up"}}]}"#, + ) + .expect("parse"); + // What ships is rendered from the same facts the gated builds used. + assert_eq!( + input + .source_unit + .parse::() + .expect("the chain's shape") + .package, + "l" + ); + assert!(input.prep_facts.is_empty()); + assert_eq!(input.setup.as_deref(), Some("struct Fixture {}")); + // A component that gave up carries nothing to read, and `delivered` skips it. + let delivered: Vec<&str> = input.delivered().map(|(name, _)| name).collect(); + assert_eq!(delivered, vec!["Farms"]); + assert!(matches!( + input.components[1].outcome, + ComponentOutcome::GaveUp + )); + } +} diff --git a/rust/autoprover-sdk/src/finalize.rs b/rust/autoprover-sdk/src/finalize.rs new file mode 100644 index 00000000..d4157ff9 --- /dev/null +++ b/rust/autoprover-sdk/src/finalize.rs @@ -0,0 +1,85 @@ +//! The full outcome set [`Backend::finalize`](crate::Backend::finalize) renders run-level +//! artifacts from. + +use serde::{Deserialize, Serialize}; + +use crate::chain::ChainData; +use crate::outcome::SkippedProperty; + +/// What one component's formalization produced, when it produced anything. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[cfg_attr(feature = "fuzz", derive(arbitrary::Arbitrary))] +#[serde(deny_unknown_fields)] +pub struct Delivered { + /// The authored spec, verbatim — the source a + /// [`DeliverableMode::Callout`](crate::descriptor::DeliverableMode::Callout) wheel assembles its + /// deliverable from. + pub artifact_text: String, + /// The validation targets this component's checks ran under, in the order they ran. The key a + /// callout-mode wheel writes its sections under: they are what the gated builds selected, + /// unlike `property_checks`, which are report rows and gate nothing. + pub targets: Vec, + /// Each property title and the names of the checks carrying it — the report's property→check + /// map. + pub property_checks: Vec<(String, Vec)>, + /// The properties the author declined to formalize, each with its justification. Disjoint from + /// `property_checks`: the publish gate rejects a mapping that claims a skipped property. + pub skipped: Vec, + #[serde(deserialize_with = "crate::required::present")] + pub unit_file: Option, + #[serde(deserialize_with = "crate::required::present")] + pub run_link: Option, +} + +/// Whether a component reached the deliverable. An enum rather than a `delivered` flag beside +/// always-present fields: a component that gave up has no spec, no targets and no checks, so there +/// is nothing for a caller to read past the name. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(tag = "status", rename_all = "snake_case")] +#[cfg_attr(feature = "fuzz", derive(arbitrary::Arbitrary))] +#[serde(deny_unknown_fields)] +pub enum ComponentOutcome { + Delivered(Delivered), + /// Formalization gave up on this component; it contributes nothing to the deliverable. + GaveUp, +} + +/// One component's line in the outcome set. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[cfg_attr(feature = "fuzz", derive(arbitrary::Arbitrary))] +#[serde(deny_unknown_fields)] +pub struct FinalizeComponent { + pub name: String, + pub outcome: ComponentOutcome, +} + +/// The complete outcome set. Everything a wheel needs to render the whole deliverable: the same +/// project facts the gated builds used (so what ships is what was checked), the compiled shared +/// setup spec, and every component's result. +#[derive(Debug, Clone, Default, Serialize, Deserialize)] +#[cfg_attr(feature = "fuzz", derive(arbitrary::Arbitrary))] +#[serde(deny_unknown_fields)] +pub struct FinalizeInput { + pub program: String, + /// As every authoring callout received it — see + /// [`AuthorInput::source_unit`](crate::authoring::AuthorInput::source_unit). + pub source_unit: ChainData, + /// As every authoring callout received it — see + /// [`AuthorInput::prep_facts`](crate::authoring::AuthorInput::prep_facts). + pub prep_facts: ChainData, + /// The compiled shared setup spec, when the wheel declared a + /// [`PhaseRole::Setup`](crate::descriptor::PhaseRole::Setup) phase. + #[serde(deserialize_with = "crate::required::present")] + pub setup: Option, + pub components: Vec, +} + +impl FinalizeInput { + /// The components that reached the deliverable, as `(name, result)`. + pub fn delivered(&self) -> impl Iterator { + self.components.iter().filter_map(|c| match &c.outcome { + ComponentOutcome::Delivered(d) => Some((c.name.as_str(), d)), + ComponentOutcome::GaveUp => None, + }) + } +} diff --git a/rust/autoprover-sdk/src/fuzz.rs b/rust/autoprover-sdk/src/fuzz.rs new file mode 100644 index 00000000..3341c00c --- /dev/null +++ b/rust/autoprover-sdk/src/fuzz.rs @@ -0,0 +1,153 @@ +//! `Arbitrary` impls for the wire types whose Rust type is wider than the protocol, so the +//! round-trip fuzzer (`tests/test_wire_roundtrip.py`) generates values the protocol actually +//! permits. A divergence it reports is then drift, not a value neither side ever sends. +//! +//! Everything else derives `Arbitrary` at its definition. What needs a hand-written impl is +//! either a field holding a `serde_json::Value` (no `Arbitrary` impl, and none would know how +//! deep to go) or a field whose wire domain is narrower than its Rust type — the cases below, +//! each of which would otherwise fail as a false positive. + +use arbitrary::{Arbitrary, Result, Unstructured}; +use serde_json::{Map, Value}; + +use crate::args::DeclaredArgs; +use crate::authoring::Authored; +use crate::chain::ChainData; +use crate::descriptor::{ + AppDescriptor, ArgSpec, ArtifactLayout, DeliverableMode, EventKind, PhaseSpec, +}; +use crate::outcome::{Outcome, Verdict}; + +/// The chain tags the host resolves against its ecosystem registry (`ChainTag`). +const ECOSYSTEMS: &[&str] = &["evm", "solana", "soroban"]; + +/// The report vocabularies a descriptor may claim (`ReportBackend`). +const REPORT_BACKENDS: &[&str] = &["prover", "foundry", "none"]; + +/// How deep a generated opaque payload nests. The host treats these as opaque either way, so +/// depth buys no coverage past the point where "it is a JSON document" is established. +const JSON_DEPTH: u32 = 2; + +/// Cap on any generated collection — a few elements reach the same code as a long list. +const MAX_ITEMS: usize = 3; + +/// A JSON scalar, minus floats: `NaN`/infinity have no JSON spelling (`serde_json` refuses to +/// serialize them), so drawing one would fail the harness rather than either side of the seam. +fn scalar(u: &mut Unstructured) -> Result { + Ok(match u.int_in_range(0..=3)? { + 0 => Value::Null, + 1 => Value::Bool(bool::arbitrary(u)?), + 2 => Value::from(i64::arbitrary(u)?), + _ => Value::String(String::arbitrary(u)?), + }) +} + +/// An arbitrary JSON document, nesting at most `depth` deeper. Weighted toward scalars so a draw +/// terminates well inside the entropy a single example carries. +fn json(u: &mut Unstructured, depth: u32) -> Result { + if depth == 0 { + return scalar(u); + } + match u.int_in_range(0..=5)? { + 0..=3 => scalar(u), + 4 => { + let mut items = Vec::new(); + for _ in 0..u.int_in_range(0..=MAX_ITEMS)? { + items.push(json(u, depth - 1)?); + } + Ok(Value::Array(items)) + } + _ => Ok(Value::Object(object(u, depth - 1)?)), + } +} + +fn object(u: &mut Unstructured, depth: u32) -> Result> { + let mut map = Map::new(); + for _ in 0..u.int_in_range(0..=MAX_ITEMS)? { + map.insert(String::arbitrary(u)?, json(u, depth)?); + } + Ok(map) +} + +/// Opaque on both sides (the wheel declares the flags, so the host has no schema for them), which +/// makes any JSON object a legal value. +impl<'a> Arbitrary<'a> for DeclaredArgs { + fn arbitrary(u: &mut Unstructured<'a>) -> Result { + Ok(object(u, JSON_DEPTH)?.into()) + } +} + +/// The project's own build-system vocabulary, opaque to this seam (`chain::ChainData`) — so, like +/// the declared flags, any JSON object is a legal value. +impl<'a> Arbitrary<'a> for ChainData { + fn arbitrary(u: &mut Unstructured<'a>) -> Result { + Ok(object(u, JSON_DEPTH)?.into()) + } +} + +/// The `model` and `unit` payloads are the ecosystem's shape, opaque to this seam. +impl<'a> Arbitrary<'a> for Authored { + fn arbitrary(u: &mut Unstructured<'a>) -> Result { + Ok(match u.int_in_range(0..=2)? { + 0 => Authored::Preflight, + 1 => Authored::Setup { + model: json(u, JSON_DEPTH)?, + units: Vec::arbitrary(u)?, + }, + _ => Authored::Component { + unit: json(u, JSON_DEPTH)?, + }, + }) + } +} + +/// `duration_seconds` is an `f64`, but a duration is finite: drawn from milliseconds so every +/// value has a JSON spelling and few enough significant digits that the comparison tests the +/// protocol rather than two languages' float formatters. +impl<'a> Arbitrary<'a> for Verdict { + fn arbitrary(u: &mut Unstructured<'a>) -> Result { + let seconds = if bool::arbitrary(u)? { + Some(u.int_in_range(0..=86_400_000)? as f64 / 1000.0) + } else { + None + }; + Ok(Verdict { + outcome: Outcome::arbitrary(u)?, + line: Option::arbitrary(u)?, + duration_seconds: seconds, + unit_file: Option::arbitrary(u)?, + detail: Option::arbitrary(u)?, + }) + } +} + +/// `ecosystem` and `backend_tag` are `String` here but closed sets on the host, which validates +/// them when it parses the descriptor — that narrowing is deliberate (a wheel claiming a tag the +/// report has no wording for should fail at load, before the run starts), so the round trip draws +/// from the sets rather than reporting every unknown tag as drift. +/// +/// `tests/test_wire_roundtrip.py::test_descriptor_rejects_unknown_tags` covers the other half: +/// that a tag outside the set is in fact rejected. +impl<'a> Arbitrary<'a> for AppDescriptor { + fn arbitrary(u: &mut Unstructured<'a>) -> Result { + Ok(AppDescriptor { + name: String::arbitrary(u)?, + header_text: String::arbitrary(u)?, + ecosystem: (*u.choose(ECOSYSTEMS)?).to_string(), + backend_tag: (*u.choose(REPORT_BACKENDS)?).to_string(), + backend_guidance: String::arbitrary(u)?, + analysis_key: String::arbitrary(u)?, + phases: Vec::::arbitrary(u)?, + args: Vec::::arbitrary(u)?, + rag_db_default: Option::arbitrary(u)?, + event_kinds: Vec::::arbitrary(u)?, + artifact_layout: ArtifactLayout::arbitrary(u)?, + deliverable_mode: DeliverableMode::arbitrary(u)?, + serialize_toolchain: bool::arbitrary(u)?, + confine_by_default: bool::arbitrary(u)?, + component_noun: Option::arbitrary(u)?, + check_noun: Option::arbitrary(u)?, + evidence_kinds: Vec::::arbitrary(u)?, + }) + } +} diff --git a/rust/autoprover-sdk/src/lib.rs b/rust/autoprover-sdk/src/lib.rs new file mode 100644 index 00000000..81b4190d --- /dev/null +++ b/rust/autoprover-sdk/src/lib.rs @@ -0,0 +1,62 @@ +//! # autoprover-sdk +//! +//! The library a Rust-based AutoProver application imports. It defines the seam +//! between a Rust backend and the generic Python pipeline +//! (`composer/pipeline/core.py`), realized over a **synchronous, JSON** FFI +//! boundary — the service-shaped design in `docs/rust-applications.md`. +//! +//! The backend is a **passive service**, not a driver: the Python pipeline owns the +//! author→compile→judge→validate loop and every LLM turn, and calls the backend's +//! callouts. Most are pure ([`Backend::descriptor`], [`Backend::target_for`], +//! [`Backend::author_prompt`], [`Backend::judge`], [`Backend::finalize`]). The +//! two gating callouts ([`Backend::compile`], [`Backend::validate`]) run the toolchain +//! directly — each spawns the `run-confined` launcher via +//! [`Workspace::run`](sandbox::Workspace::run) — and BLOCK; the host calls them off the event +//! loop (`asyncio.to_thread`) while the wheel releases the GIL. There is no `async`/`pyo3-async` +//! bridge and no `Command`/`Observation` resume protocol on the Rust side. +//! +//! An application implements [`Backend`] and calls [`export_app!`] to emit the PyO3 +//! module the Python host loads. +//! +//! ## Where things live +//! +//! Types are addressed through the module that owns them (`descriptor::AppDescriptor`, +//! `outcome::Verdict`), grouped by which part of the seam they belong to: +//! +//! * [`descriptor`] — the declaration the host reads at load time. +//! * [`args`] — the run's resolved inputs (project root, program, declared flags). +//! * [`chain`] — the payload this seam carries without a schema: what the analyzed project's own +//! build system calls things. +//! * [`authoring`] — what the host sends into a callout, and the prompt it gets back. +//! * [`outcome`] — the compile verdict, the property→check map, the per-check verdicts. +//! * [`finalize`] — the full outcome set the run-level deliverable is rendered from. +//! * [`prep`] — pure plans the host executes for the wheel (workspace prep, sandbox grants). +//! * [`sandbox`] — where a blocking callout runs its toolchain, and how it spawns. +//! * [`ffi`] — the JSON-string boundary [`export_app!`] wraps, and the +//! [`CalloutError`](ffi::CalloutError) envelope a callout returns instead of its payload. + +pub mod args; +pub mod authoring; +pub mod backend; +pub mod chain; +pub mod descriptor; +pub mod export; +pub mod ffi; +pub mod finalize; +pub mod outcome; +pub mod prep; +pub mod required; +pub mod sandbox; + +/// Value generation for the wire round-trip fuzzer, behind the `fuzz` feature so it never enters +/// a shipped wheel. Driven by the `wire-echo` binary. +#[cfg(feature = "fuzz")] +pub mod fuzz; + +/// The trait an application implements — re-exported because it is what every app names, and +/// [`backend`] holds nothing else. +pub use backend::Backend; + +/// Re-exported so [`export_app!`] can reference `$crate::pyo3::…`; an app crate +/// still depends on pyo3 directly to enable `extension-module` / `abi3-py312`. +pub use pyo3; diff --git a/rust/autoprover-sdk/src/outcome.rs b/rust/autoprover-sdk/src/outcome.rs new file mode 100644 index 00000000..a1764142 --- /dev/null +++ b/rust/autoprover-sdk/src/outcome.rs @@ -0,0 +1,215 @@ +//! What the gating callouts hand back: the compile verdict, the report's property→check map, +//! and the per-check validation outcomes. + +use serde::{Deserialize, Serialize}; + +/// The outcome of `compile`. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(tag = "status", rename_all = "snake_case")] +#[cfg_attr(feature = "fuzz", derive(arbitrary::Arbitrary))] +#[serde(deny_unknown_fields)] +pub enum CompileResult { + Ok, + Failed { errors: String }, +} + +/// A property the author declined to formalize, with its justification. Carried into +/// [`Delivered`](crate::finalize::Delivered) so the deliverable and the report can both say what was +/// left out and why — a property that is absent for a stated reason is a different thing from one +/// that was silently dropped. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[cfg_attr(feature = "fuzz", derive(arbitrary::Arbitrary))] +#[serde(deny_unknown_fields)] +pub struct SkippedProperty { + pub property_title: String, + pub reason: String, +} + +/// One check the **author declared**: the backend's name for a runnable verification — a CVL rule, a +/// foundry test, a tagged fuzz assertion. A check yields a [`Verdict`] and becomes one row of the +/// report. +/// +/// `properties` is what the author declared this check verifies — the mapping's own claim, carried +/// verbatim rather than guessed by the wheel, and never empty (a check exists *because* something +/// claimed it). Usually one title; several when one rule discharges several related invariants. It +/// is here because some checkers speak in properties rather than in check names — Crucible tags each +/// assertion message with its property title, so this is what lets it place a counterexample — +/// while a backend whose checker reports per check (the Prover, forge) can ignore it. +/// +/// `target` names the [`Target`] this check runs under — **one invocation of the checker**. Several +/// checks may share one (e.g. Crucible puts a component's whole property set in a single fuzz +/// target), so the host runs it once and the backend attributes the outcome back to each check. +/// `None` ⇒ the check is its own target (one invocation per check, the default), and is what +/// [`Backend::target_for`](crate::Backend::target_for) answers by default. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[cfg_attr(feature = "fuzz", derive(arbitrary::Arbitrary))] +#[serde(deny_unknown_fields)] +pub struct Check { + pub name: String, + pub properties: Vec, + #[serde(deserialize_with = "crate::required::present")] + pub target: Option, +} + +impl Check { + /// The validation target this check runs under — its own `name` unless it shares a target with + /// others. + pub fn target_or_name(&self) -> &str { + self.target.as_deref().unwrap_or(&self.name) + } +} + +/// **One invocation of the checker** — one build + one run — and the checks that invocation covers, +/// which are the checks [`Backend::validate`](crate::Backend::validate) must return a verdict for. +/// +/// Targets are how *running* is grouped; [`Check`]s are how *reporting* is. A target always sits +/// inside a single unit's session (the [`AuthorInput::unit`](crate::authoring::AuthorInput::unit) +/// being formalized), so the three nest: one unit's checks partition into its targets. That is the +/// point of the split — a backend that fuzzes a whole property set in one campaign pays for one +/// build and one run, and still reports a row per property. +/// +/// The host computes the grouping from the checks the author declared, each targeted by +/// [`Backend::target_for`](crate::Backend::target_for) (it decides what to run, and in what order), +/// so it hands the answer over rather than leaving each backend to recover it by filtering a set it +/// would have to rebuild. A target name is only meaningful within its session: two units naming the +/// same target each run their own. +#[derive(Debug, Clone, Default, Serialize, Deserialize)] +#[cfg_attr(feature = "fuzz", derive(arbitrary::Arbitrary))] +#[serde(deny_unknown_fields)] +pub struct Target { + /// The target's name — what the backend selects when it invokes the checker (Crucible: the + /// unit's harness fn, which is also its Cargo feature). + pub name: String, + /// The checks this run must produce a verdict for. Usually one; several when a backend checks + /// a whole property set in one run. + pub checks: Vec, +} + +impl Target { + /// The same verdict for every covered check — what a run that concluded one thing about the + /// whole target produces (it errored; it timed out). + /// + /// Note what is NOT such a conclusion: a run that *finished cleanly*. "Nothing was refuted" is + /// one fact about the target, but [`Outcome::Good`] per check is a claim about each check + /// individually, and a check the run never exercised has not been shown to hold — see the + /// verdict contract on [`Backend::validate`](crate::Backend::validate). Reach for + /// [`Target::verdicts`] there. + pub fn all(&self, outcome: Outcome, detail: Option) -> ValidateOutcome { + self.verdicts(|_| Verdict::with_outcome(outcome).with_detail(detail.clone())) + } + + /// A verdict per covered check, keyed by check name so a backend never spells one itself. + pub fn verdicts(&self, mut of: impl FnMut(&Check) -> Verdict) -> ValidateOutcome { + ValidateOutcome::Verdicts { + verdicts: CheckVerdicts( + self.checks + .iter() + .map(|c| (c.name.clone(), of(c))) + .collect(), + ), + } + } +} + +/// What a target's run concluded, as `(check_name, verdict)` for every check the target covers. +/// +/// The payload is private, so the only way a backend builds one is [`Target::all`] or +/// [`Target::verdicts`] — both of which take the names from the target's own checks. A backend +/// therefore attributes its run to the checks it was handed instead of naming them: it can neither +/// invent a check the target does not cover nor leave one of them without a verdict, and the host +/// resolves each name back to the [`Check`] it sent. +#[derive(Debug, Clone, Default, Serialize, Deserialize)] +#[serde(transparent)] +#[cfg_attr(feature = "fuzz", derive(arbitrary::Arbitrary))] +pub struct CheckVerdicts(Vec<(String, Verdict)>); + +impl CheckVerdicts { + /// Each covered check's name and what it concluded. + pub fn iter(&self) -> impl Iterator { + self.0 + .iter() + .map(|(name, verdict)| (name.as_str(), verdict)) + } +} + +/// The result of `validate` — the fused build+check for one validation **target**. Either the +/// build failed (so the whole spec must be re-authored — the build is shared), or it built and +/// produced a `Verdict` **per check the target covers** (`(check_name, verdict)`). A target may +/// cover several checks (e.g. Crucible runs every invariant in one target), and the backend — which +/// owns its own result/failure format — attributes the run to those checks; the host records the +/// verdicts verbatim (it does no verdict logic). The build gate is fused in here rather than run as +/// a separate `compile` dry-run, so a component pays for one build (docs/rust-applications.md §4.4). +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(tag = "kind", rename_all = "snake_case")] +#[cfg_attr(feature = "fuzz", derive(arbitrary::Arbitrary))] +#[serde(deny_unknown_fields)] +pub enum ValidateOutcome { + BuildFailed { errors: String }, + Verdicts { verdicts: CheckVerdicts }, +} + +/// What one check concluded — the report's backend-agnostic vocabulary (mirrors +/// `composer…report.schema.Outcome`), which every backend's native status maps into. The +/// human-facing wording ("No counterexample" vs "Verified") is picked at render time from the +/// application's `backend_tag`, so a backend never spells it out here. +/// +/// An enum rather than a free string: the host validates this field against the same closed set, so +/// a typo fails to compile here instead of reaching a report row as an unexplained `UNKNOWN`. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "UPPERCASE")] +#[cfg_attr(feature = "fuzz", derive(arbitrary::Arbitrary))] +pub enum Outcome { + /// The property holds. + Good, + /// The property is violated — `Verdict::detail` should carry the counterexample. + Bad, + /// The check errored out without reaching a verdict. + Error, + /// The check ran out of time without reaching a verdict. + Timeout, + /// No conclusive result. + Unknown, +} + +/// One check's outcome (mirrors `composer…report.collect.Verdict`). +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Verdict { + pub outcome: Outcome, + #[serde(deserialize_with = "crate::required::present")] + pub line: Option, + #[serde(deserialize_with = "crate::required::present")] + pub duration_seconds: Option, + #[serde(deserialize_with = "crate::required::present")] + pub unit_file: Option, + /// Human-readable explanation of a non-GOOD outcome — the failure detail (a counterexample / + /// assertion message) for a `BAD`, or the error text for an `ERROR`. Surfaced live and persisted + /// to the report so a verdict is self-explaining (otherwise a bare `BAD` gives no clue why). + #[serde(deserialize_with = "crate::required::present")] + pub detail: Option, +} + +impl Verdict { + /// A bare verdict: just the outcome, no diagnostics. Set the fields you have on the result. + pub fn with_outcome(outcome: Outcome) -> Self { + Verdict { + outcome, + line: None, + duration_seconds: None, + unit_file: None, + detail: None, + } + } + + /// A failing verdict carrying its explanation — the shape a backend almost always wants for a + /// `Bad` or an `Error`, since a bare one gives a reader no clue why. + pub fn detailed(outcome: Outcome, detail: impl Into) -> Self { + Verdict::with_outcome(outcome).with_detail(Some(detail.into())) + } + + /// This verdict with `detail` when there is one — for a caller holding the `Option` a parsed + /// tool output gives it, rather than one deciding between two constructors. + pub fn with_detail(self, detail: Option) -> Self { + Verdict { detail, ..self } + } +} diff --git a/rust/autoprover-sdk/src/prep.rs b/rust/autoprover-sdk/src/prep.rs new file mode 100644 index 00000000..8f3be208 --- /dev/null +++ b/rust/autoprover-sdk/src/prep.rs @@ -0,0 +1,55 @@ +//! Pure declarations the *host* executes on the wheel's behalf — the workspace plan and the +//! sandbox grants unioned into the Python-authored policy. Nothing here runs a command line. + +use serde::{Deserialize, Serialize}; +use std::collections::BTreeMap; + +use crate::chain::ChainData; + +/// A pure plan for preparing the workspace before formalization (Crucible: place the harness +/// manifest, warm its deps, build the program). The wheel *declares* the plan; the **host executes +/// it**, so the standard network posture holds without the wheel touching a command line: +/// dependency fetches run *unconfined* (network, no untrusted code), and anything that compiles runs +/// *confined + offline* (`docs/command-sandbox.md` §5). This keeps warming out of confinement — the +/// codebase never gives a confined process network — while still letting a pure-Rust app own its +/// layout. +/// +/// Two halves, split by who can execute them: writing files is the same everywhere, while preparing +/// a *project* means driving a build system the host does not understand, which only that chain's +/// registered `ProjectToolchain` can do. +#[derive(Debug, Clone, Default, Serialize, Deserialize)] +#[cfg_attr(feature = "fuzz", derive(arbitrary::Arbitrary))] +#[serde(deny_unknown_fields)] +pub struct WorkspacePrep { + /// Files to write under the workdir (path-confined) before anything else — e.g. the harness + /// manifest, whose contents only the wheel knows. Contents only; no command line. + pub files: BTreeMap, + /// What the chain's `ProjectToolchain` should do beyond writing [`WorkspacePrep::files`]: warm a + /// dependency cache, build the program, derive a client from it. Empty asks for nothing, and a + /// plan that only places files is complete once they are written. + /// + /// [Chain-shaped](ChainData) — build one with [`ChainData::of`] from the request type your + /// chain's support crate defines (Solana: `{warm_dirs, build_program, idl_dest}`). The host + /// forwards it without a schema and only asks whether it is empty, which is what keeps a new + /// ecosystem a registration rather than a field here. Whatever the toolchain establishes comes + /// back on every later callout as + /// [`AuthorInput::prep_facts`](crate::authoring::AuthorInput::prep_facts). + /// + /// A request that cannot be carried out is a hard error, not a silent skip: a wheel asks only + /// when it cannot proceed without the result. + pub toolchain_request: ChainData, +} + +/// Extra sandbox grants a wheel needs unioned into the host-authored policy (Crucible: the +/// crucible checkout + the `crucible` binary dir as read-only). Pure data — the wheel declares +/// grants, Python decides the policy; the wheel never invents confinement. +#[derive(Debug, Clone, Default, Serialize, Deserialize)] +#[cfg_attr(feature = "fuzz", derive(arbitrary::Arbitrary))] +#[serde(deny_unknown_fields)] +pub struct SandboxGrants { + /// Extra read-only paths. + pub extra_ro: Vec, + /// Extra env variable *names* to pass through confinement — the host unions these into its + /// passthrough list, so a value is inherited from the ambient environment, never supplied here. + pub extra_env: Vec, +} diff --git a/rust/autoprover-sdk/src/required.rs b/rust/autoprover-sdk/src/required.rs new file mode 100644 index 00000000..f0a5bda2 --- /dev/null +++ b/rust/autoprover-sdk/src/required.rs @@ -0,0 +1,28 @@ +//! The one piece of machinery the protocol's "every field is present" rule needs. +//! +//! Both halves of this seam ship together, so a payload missing a field is never version skew — it +//! is a mirror that drifted. Every wire type therefore requires every field and rejects unknown +//! ones (`#[serde(deny_unknown_fields)]`), which turns a one-sided field into an error at the first +//! callout, naming it, instead of a default that reads as data forever. +//! +//! Plain fields get that for free: without `#[serde(default)]`, serde already fails on a missing +//! key. `Option` does not — serde deserializes a missing field of that type as `None` whether or +//! not the attribute is there, because its missing-field handling deserializes from a unit-like +//! deserializer that `Option` accepts. [`present`] is how such a field opts back in: it routes +//! through `deserialize_with`, which serde cannot satisfy from a missing key, so absence is an +//! error and an empty value must be spelled `null`. + +use serde::{Deserialize, Deserializer}; + +/// An `Option` field that must still be **present** on the wire — `null` when it holds nothing. +/// +/// Used as `#[serde(deserialize_with = "crate::required::present")]`. Pair it with *no* +/// `skip_serializing_if`, so the key is always written and the two sides never have to agree on +/// whether an absent key and a null one mean the same thing. +pub fn present<'de, D, T>(deserializer: D) -> Result, D::Error> +where + D: Deserializer<'de>, + T: Deserialize<'de>, +{ + Option::::deserialize(deserializer) +} diff --git a/rust/autoprover-sdk/src/sandbox.rs b/rust/autoprover-sdk/src/sandbox.rs new file mode 100644 index 00000000..47bfd2d0 --- /dev/null +++ b/rust/autoprover-sdk/src/sandbox.rs @@ -0,0 +1,297 @@ +//! Where a blocking callout runs its toolchain: the workdir, the Python-authored confinement +//! wrapper, and the launcher helper that spawns behind it. + +use serde::{Deserialize, Serialize}; +use std::collections::BTreeMap; +use std::ffi::OsStr; +use std::io::Read; +use std::path::{Path, PathBuf}; +use std::process::{Child, Command, ExitStatus, Stdio}; +use std::thread::JoinHandle; +use std::time::{Duration, Instant}; + +/// The confinement wrapper for a command, authored by Python +/// (`SandboxConfig.backend_spec`) and passed to `compile`/`validate`. The backend never +/// invents policy or names a sandbox mechanism: Python owns the confinement *intent* and +/// translates it into `argv_prefix`, an opaque argv the backend simply prepends to its +/// command — `[*argv_prefix, program, *args]` (see [`Workspace::run`]). +/// +/// `argv_prefix` is **empty** for a passthrough (`provider="none"`) spec — the command runs +/// directly (the trusted path). Otherwise it is a full `run-confined --` wrapper +/// (mirrors `composer/sandbox/launcher.py::LauncherProvider.argv_prefix`); its first element +/// is the launcher binary. Because the prefix is opaque, swapping the sandbox mechanism never +/// changes this shape. +#[derive(Debug, Clone, Default, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +#[cfg_attr(feature = "fuzz", derive(arbitrary::Arbitrary))] +pub struct Sandbox { + pub argv_prefix: Vec, + pub timeout_s: u64, +} + + +/// The captured result of a (confined) command. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct CommandOutput { + pub exit_code: i32, + pub stdout: String, + pub stderr: String, +} + +/// Exit code synthesized when the program isn't found (mirrors shells' 127). +const NOT_FOUND_EXIT: i32 = 127; + +/// Reject absolute paths / `..` traversal (mirrors `composer.sandbox.command._confined_target`). +fn confined_join(workdir: &Path, rel: &str) -> Result { + use std::path::Component; + let p = Path::new(rel); + if p.is_absolute() || p.components().any(|c| matches!(c, Component::ParentDir)) { + return Err(format!("unsafe file path {rel:?}: absolute or traverses outside the workdir")); + } + Ok(workdir.join(p)) +} + +/// Where one blocking callout runs its toolchain: the directory, and the confinement to run +/// behind. The two always travel together — every command a backend runs needs both — so +/// [`Backend::compile`](crate::Backend::compile) and [`Backend::validate`](crate::Backend::validate) +/// are handed this rather than a loose pair. +#[derive(Debug, Clone)] +pub struct Workspace { + /// The workdir: where files are materialized and the command runs. Also the root every path a + /// backend reads back (a checker's output file, say) is resolved against. + pub dir: PathBuf, + /// The confinement wrapper to run behind. + pub sandbox: Sandbox, +} + +impl Workspace { + /// Materialize `files` into the workdir (path-confined), then run `program args` there behind + /// the sandbox's `argv_prefix` — i.e. `[*argv_prefix, program, *args]` (or `program args` + /// directly, when the prefix is empty). Blocks on the child; the host already calls the + /// blocking callouts with the GIL released. Enforces `sandbox.timeout_s` by SIGKILL of the + /// child's process group, so descendants die with the leader. + /// + /// The **command line (`program`/`args`) is authored by the trusted backend**; only file + /// *contents* may derive from the LLM (`docs/command-sandbox.md` §2, + /// `docs/rust-applications.md` §8). When present, the prefix's `run-confined` confines *itself* + /// (Landlock+seccomp+rlimits+env scrub) and `execve`s the tool. + pub fn run( + &self, + program: &str, + args: I, + files: &BTreeMap, + ) -> Result + where + I: IntoIterator, + S: AsRef, + { + run_confined(&self.sandbox, program, args, files, &self.dir) + } +} + +/// How a spawned command finished: it exited, or the wall-clock timeout fired. +enum ChildStatus { + Exited(ExitStatus), + TimedOut, +} + +/// The body of [`Workspace::run`], over the parts rather than the bundle. +fn run_confined( + sandbox: &Sandbox, + program: &str, + args: I, + files: &BTreeMap, + workdir: &Path, +) -> Result +where + I: IntoIterator, + S: AsRef, +{ + materialize_files(workdir, files)?; + let mut cmd = confined_command(sandbox, program, args, workdir); + let mut child = match cmd.spawn() { + Ok(c) => c, + Err(e) if e.kind() == std::io::ErrorKind::NotFound => { + return Ok(not_found(cmd.get_program())); + } + Err(e) => return Err(e.to_string()), + }; + let (t_out, t_err) = pipe_readers(&mut child); + let timeout_s = sandbox.timeout_s.max(1); + let status = wait_with_timeout(&mut child, timeout_s)?; + let stdout = lossy_utf8(t_out); + let stderr = lossy_utf8(t_err); + Ok(match status { + ChildStatus::Exited(st) => CommandOutput { + exit_code: st.code().unwrap_or(-1), + stdout, + stderr, + }, + ChildStatus::TimedOut => CommandOutput { + exit_code: -1, + stdout, + stderr: format!("{stderr}\ncommand timed out after {timeout_s}s"), + }, + }) +} + +/// Write `files` into `workdir`, rejecting absolute / `..` paths. Contents may be +/// LLM-derived; the command line is not. +fn materialize_files(workdir: &Path, files: &BTreeMap) -> Result<(), String> { + std::fs::create_dir_all(workdir).map_err(|e| e.to_string())?; + for (rel, contents) in files { + let target = confined_join(workdir, rel)?; + if let Some(parent) = target.parent() { + std::fs::create_dir_all(parent).map_err(|e| e.to_string())?; + } + std::fs::write(&target, contents).map_err(|e| e.to_string())?; + } + Ok(()) +} + +/// `[*argv_prefix, program, *args]`, or `program args` when the prefix is empty. +/// Owns a process group so the timeout SIGKILL takes descendants with the leader. +/// `run-confined` execve's in place, so the group leader *is* the command; cargo/rustc +/// children inherit the group unless they leave it. +fn confined_command(sandbox: &Sandbox, program: &str, args: I, workdir: &Path) -> Command +where + I: IntoIterator, + S: AsRef, +{ + let mut cmd = match sandbox.argv_prefix.split_first() { + Some((bin, rest)) => { + // The prefix ends at its `--`; the wrapped command follows it. + let mut cmd = Command::new(bin); + cmd.args(rest).arg(program); + cmd + } + None => Command::new(program), + }; + cmd.args(args) + .current_dir(workdir) + .stdin(Stdio::null()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()); + #[cfg(unix)] + { + use std::os::unix::process::CommandExt; + cmd.process_group(0); + } + cmd +} + +fn not_found(program: &OsStr) -> CommandOutput { + CommandOutput { + exit_code: NOT_FOUND_EXIT, + stdout: String::new(), + stderr: format!("{}: not found", program.to_string_lossy()), + } +} + +/// Drain stdout/stderr on side threads so a full pipe cannot deadlock the waiter. +fn pipe_readers(child: &mut Child) -> (JoinHandle>, JoinHandle>) { + let out = child.stdout.take().expect("piped stdout"); + let err = child.stderr.take().expect("piped stderr"); + (read_pipe(out), read_pipe(err)) +} + +fn read_pipe(mut pipe: impl Read + Send + 'static) -> JoinHandle> { + std::thread::spawn(move || { + let mut s = Vec::new(); + let _ = pipe.read_to_end(&mut s); + s + }) +} + +fn lossy_utf8(buf: JoinHandle>) -> String { + String::from_utf8_lossy(&buf.join().unwrap_or_default()).into_owned() +} + +fn wait_with_timeout(child: &mut Child, timeout_s: u64) -> Result { + let deadline = Instant::now() + Duration::from_secs(timeout_s); + loop { + match child.try_wait().map_err(|e| e.to_string())? { + Some(st) => return Ok(ChildStatus::Exited(st)), + None if Instant::now() >= deadline => { + kill_spawned(child); + let _ = child.wait(); + return Ok(ChildStatus::TimedOut); + } + None => std::thread::sleep(Duration::from_millis(50)), + } + } +} + +/// Kill the spawned command. On Unix this is the process group we created at spawn +/// (`pgid == pid`); `Child::kill` would leave descendants running. +fn kill_spawned(child: &mut Child) { + #[cfg(unix)] + { + let pid = child.id() as i32; + // Negative pid = process group. Kill the group *before* wait so we cannot + // reap the leader and then killpg a reused pid. + let _ = unsafe { libc::kill(-pid, libc::SIGKILL) }; + } + #[cfg(not(unix))] + { + let _ = child.kill(); + } +} + +#[cfg(all(test, unix))] +mod tests { + use super::*; + use std::time::{Duration, Instant}; + + fn unique_workdir() -> PathBuf { + std::env::temp_dir().join(format!( + "ap-sandbox-pg-{}-{}", + std::process::id(), + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .expect("clock") + .as_nanos() + )) + } + + fn pid_gone(pid: i32, within: Duration) -> bool { + let deadline = Instant::now() + within; + loop { + if unsafe { libc::kill(pid, 0) } != 0 { + return true; + } + if Instant::now() >= deadline { + return false; + } + std::thread::sleep(Duration::from_millis(20)); + } + } + + #[test] + fn timeout_kills_the_process_group() { + let dir = unique_workdir(); + std::fs::create_dir_all(&dir).unwrap(); + let sandbox = Sandbox { + argv_prefix: Vec::new(), + timeout_s: 1, + }; + let out = run_confined( + &sandbox, + "sh", + ["-c", "sleep 100 & echo $! > child.pid; exec sleep 100"], + &BTreeMap::new(), + &dir, + ) + .unwrap(); + assert_eq!(out.exit_code, -1, "{}", out.stderr); + assert!(out.stderr.contains("timed out"), "{}", out.stderr); + let child_pid: i32 = std::fs::read_to_string(dir.join("child.pid")) + .unwrap() + .trim() + .parse() + .expect("grandchild pid"); + let gone = pid_gone(child_pid, Duration::from_secs(2)); + let _ = std::fs::remove_dir_all(&dir); + assert!(gone, "grandchild {child_pid} still alive after timeout"); + } +} diff --git a/rust/example-app/Cargo.toml b/rust/example-app/Cargo.toml new file mode 100644 index 00000000..f217c196 --- /dev/null +++ b/rust/example-app/Cargo.toml @@ -0,0 +1,18 @@ +[package] +name = "example-app" +edition.workspace = true +version.workspace = true +license.workspace = true +description = "Demonstration AutoProver application in Rust (the 'echo prover'), built into a Python wheel." + +[lib] +# MUST match the `export_app!` module ident and the maturin module name. +name = "echoprover" +crate-type = ["cdylib"] + +[dependencies] +autoprover-sdk = { path = "../autoprover-sdk" } +serde = { workspace = true } +serde_json = { workspace = true } +# `extension-module` (don't link libpython) + `abi3-py312` (one wheel for cp312+). +pyo3 = { workspace = true, features = ["extension-module", "abi3-py312"] } diff --git a/rust/example-app/pyproject.toml b/rust/example-app/pyproject.toml new file mode 100644 index 00000000..2b54242a --- /dev/null +++ b/rust/example-app/pyproject.toml @@ -0,0 +1,30 @@ +[build-system] +requires = ["maturin>=1.5,<2.0"] +build-backend = "maturin" + +[project] +name = "echoprover" +version = "0.1.0" +description = "Demonstration AutoProver application implemented in Rust." +requires-python = ">=3.12" +classifiers = ["Programming Language :: Rust"] + +[tool.maturin] +# The compiled module is imported as `echoprover` (matches `export_app!(echoprover, …)`). +module-name = "echoprover" +# abi3 wheel — one artifact for cp312+. +features = ["pyo3/extension-module"] + +[tool.uv] +# uv revalidates this path dep on every `uv run` / `uv sync`, but by default only +# watches `pyproject.toml` — which never changes when you edit Rust. These keys +# make a source change invalidate the built wheel, so `uv run pytest` recompiles +# on its own and no manual `maturin develop` step exists. +cache-keys = [ + { file = "src/**/*.rs" }, + { file = "Cargo.toml" }, + { file = "../autoprover-sdk/src/**/*.rs" }, + { file = "../autoprover-sdk/Cargo.toml" }, + { file = "../Cargo.toml" }, + { file = "../Cargo.lock" }, +] diff --git a/rust/example-app/src/lib.rs b/rust/example-app/src/lib.rs new file mode 100644 index 00000000..98bc4c6f --- /dev/null +++ b/rust/example-app/src/lib.rs @@ -0,0 +1,119 @@ +//! The "echo prover" — a minimal, self-contained demonstration of a Rust-based +//! AutoProver [`Backend`] on `autoprover-sdk`. It is intentionally not a real +//! verifier: it authors a "spec" from an LLM turn, treats compilation as a no-op, +//! and treats reading the spec as its whole checker — enough to exercise the Python +//! host + FFI round-trip (descriptor, author_prompt, compile, validate) without any +//! real toolchain. A production backend keeps this exact shape and swaps the callouts +//! for real ones (see `docs/rust-applications.md`). + +use autoprover_sdk::authoring::{AuthorInput, Prompt}; +use autoprover_sdk::descriptor::{ + default_evidence_kinds, AppDescriptor, ArgDefault, ArgSpec, ArtifactLayout, DeliverableMode, + EventKind, PhaseRole, PhaseSpec, +}; +use autoprover_sdk::outcome::{CompileResult, Outcome, Target, ValidateOutcome, Verdict}; +use autoprover_sdk::sandbox::Workspace; +use autoprover_sdk::Backend; + +struct EchoApp; + +impl Backend for EchoApp { + fn descriptor(&self) -> AppDescriptor { + AppDescriptor { + name: "echoprover".to_string(), + header_text: "Echo Prover (Rust demo) | AutoProver".to_string(), + ecosystem: "evm".to_string(), + // Must be a tag the *report* knows (`ReportBackend`: prover | foundry | none) — it + // picks the outcome wording, and the Python host validates it when it parses this + // descriptor. The demo has no vocabulary of its own, so it borrows "prover"; a real + // backend adds its own literal to `ReportBackend` instead. + backend_tag: "prover".to_string(), + backend_guidance: "These properties are checked by the echo backend, a demo that \ + accepts any well-formed spec. Feel free to state universal properties." + .to_string(), + analysis_key: "echoprover-analysis".to_string(), + phases: vec![ + PhaseSpec::step("analysis", "System Analysis", 0, PhaseRole::Analysis), + PhaseSpec::step("extraction", "Property Extraction", 1, PhaseRole::Extraction), + // A phase that only groups (cf. autoprove's harness/autosetup). + PhaseSpec::grouping("solving", "Solving", 2), + PhaseSpec::step("formalization", "Formalization", 3, PhaseRole::Formalization), + PhaseSpec::step("report", "Report", 4, PhaseRole::Report), + ], + args: vec![ArgSpec { + flag: "--echo-tag".to_string(), + help: "An arbitrary tag stamped into the echo spec.".to_string(), + default: ArgDefault::Str { value: Some("demo".to_string()) }, + required: false, + }], + rag_db_default: None, + event_kinds: vec![EventKind::log("solver_line", "Solver")], + artifact_layout: ArtifactLayout { + deliverable_dir: "certora/echo".into(), + internal_dir: ".certora_internal/echo".into(), + report_dir: "certora/echo/reports".into(), + artifact_dir: "certora/echo/specs".into(), + artifact_prefix: "echospec".into(), + artifact_extension: "espec".into(), + property_suffix: "property_rules".into(), + }, + // A simple per-component wheel: nothing to gate ahead of analysis (there is no + // workspace to prepare and `compile` is a no-op) and no shared setup — neither role is + // claimed by a phase above — plus one file per component and no toolchain + // confinement/serialization. All defaults. + deliverable_mode: DeliverableMode::PerComponent, + serialize_toolchain: false, + confine_by_default: false, + component_noun: None, + // The demo authors `rule_` checks, so that is what its prompts should call them. + check_noun: Some("rule".into()), + evidence_kinds: default_evidence_kinds(), + } + } + + fn author_prompt(&self, input: &AuthorInput) -> Prompt { + let titles: Vec<&str> = input.props.iter().map(|p| p.title.as_str()).collect(); + Prompt { + system: None, + instruction: format!( + "Author a spec with a rule per property: {}. Declare each rule on its own line as \ + `rule :`, and map every property to the rules that verify it.", + titles.join(", ") + ), + } + } + + fn compile(&self, _input: &AuthorInput, _spec: Option<&str>, _ws: &Workspace) -> CompileResult { + // The demo accepts any well-formed spec — no build gate. + CompileResult::Ok + } + + fn validate( + &self, + _input: &AuthorInput, + spec: &str, + target: &Target, + _ws: &Workspace, + ) -> ValidateOutcome { + // The check names are the author's, so a run has to hold them to the artifact (the verdict + // contract on `Backend::validate`). This wheel has no toolchain, so reading the spec IS its + // checker — `rule :` is the whole language. A real backend must corroborate with + // something its checker observed; reading source text for a name that *looks* like a check + // would not do, since a name in a comment or in dead code reads exactly the same. + let declared: Vec<&str> = spec + .lines() + .filter_map(|l| l.trim().strip_prefix("rule ")?.split(':').next()) + .map(str::trim) + .collect(); + target.verdicts(|c| { + if declared.contains(&c.name.as_str()) { + Verdict::with_outcome(Outcome::Good) + } else { + Verdict::with_outcome(Outcome::Error) + .with_detail(Some(format!("the spec declares no rule named `{}`", c.name))) + } + }) + } +} + +autoprover_sdk::export_app!(echoprover, EchoApp); diff --git a/rust/run-confined/pyproject.toml b/rust/run-confined/pyproject.toml new file mode 100644 index 00000000..43ea8ba8 --- /dev/null +++ b/rust/run-confined/pyproject.toml @@ -0,0 +1,31 @@ +[build-system] +requires = ["maturin>=1.5,<2.0"] +build-backend = "maturin" + +# `run-confined` is a plain binary, not an extension module — this pyproject exists +# only so the venv can build and carry it: maturin's `bin` bindings put the compiled +# binary in the wheel's scripts dir, i.e. `.venv/bin/run-confined`, which is exactly +# where `composer.sandbox.launcher._resolve_binary`'s PATH probe looks. `uv sync` +# therefore builds the launcher, and nobody has to run `cargo build` by hand. +# +# Linux-only (Landlock + seccomp); the dependency edge in the root pyproject carries +# a `sys_platform == 'linux'` marker so a macOS sync doesn't try to compile it. +[project] +name = "run-confined" +version = "0.1.0" +description = "Trusted launcher that confines a command (Landlock + seccomp + rlimits) and execs it." +requires-python = ">=3.12" +classifiers = ["Programming Language :: Rust"] + +[tool.maturin] +bindings = "bin" + +[tool.uv] +# See rust/example-app/pyproject.toml — without these, editing the launcher's Rust +# would not invalidate the built wheel. +cache-keys = [ + { file = "src/**/*.rs" }, + { file = "Cargo.toml" }, + { file = "../Cargo.toml" }, + { file = "../Cargo.lock" }, +] diff --git a/summarization_detector/README.md b/summarization_detector/README.md new file mode 100644 index 00000000..d6e942fe --- /dev/null +++ b/summarization_detector/README.md @@ -0,0 +1,62 @@ +# summarization-target detector + +A standalone AutoProver tool that, from **one prover run**, ranks the functions worth summarizing — a +candidate list with *why* each is prover-hostile, *where* it can be summarized, and (for a curated +public-library match) a suggested summary. It decides *what* to summarize, **not** *how*: the summarization +strategy (per-function over-approximation, a whole-contract symbolic model, …) and the actual summary text +are the downstream generator's job (curated summaries, the symbolic-model tool, or the CVL_GEN agent). A +caller runs it after a slow/timeout (or sanity) run to summarize the expensive functions before paying for +them. + +It is self-contained: it reads the prover's difficulty report via its own `difficulty` module and uses +`certora_autosetup` (the solc AST reader). + +## Signals + +1. **Nonlinear (SMT phase)** — the `difficulty` module: functions whose inlined body contributes nonlinear ops. +2. **Hashing / encoding (build phase)** — a solc-AST walk (`scan_ast`): `keccak256`/`sha256`/`ecrecover` + with `abi.encode*` context; assembly is excluded (Yul isn't a Solidity `FunctionCall`), and the input + length class (dynamic vs fixed-size) is used to rank. Invisible to the difficulty report. +3. **Resolved-expensive external** — a nonlinear hotspot whose owning contract isn't the CUT. + +A **reachability-from-the-main-contract gate** prunes signal-2 candidates the CUT can't reach, using the +solc AST's internal call edges plus the prover's `externalCallGraph.json` (resolved links + dispatch). + +## Usage + +The one input is a prover-run **URL** — it fetches the sources + conf, derives the main contract (the +conf's `verify` field), generates the AST, and pulls the difficulty report: + +``` +python -m summarization_detector --url https://.../output/// +``` + +Optional: `--cut` (override the derived contract), `--solc-dir` (so the conf's `solcN.NN` resolves), +`--work-dir`, `--external-call-graph` (else auto-found in the fetched tree), `--include-dependencies`. + +Offline path (when you already have the artifacts instead of a URL): + +``` +python -m summarization_detector --cut Router --conf run.conf # AST-only hashing signal +python -m summarization_detector --cut Router --ast .asts.json \ + --job-url https://.../ --external-call-graph Reports/externalCallGraph.json +``` + +`externalCallGraph.json` is emitted by the prover (EVMVerifier `ExternalCallSiteCollector`) at scene +setup — after call resolution, before the per-rule optimize pass — and enables the reachability gate. + +## difficulty_profile — post-hoc: where did the prover time actually go? + +`detect.py` predicts what to summarize *before* the rules exist. `difficulty_profile.py` is its +post-hoc counterpart: from a completed (esp. timed-out) run it reads the prover's own difficulty tree +per slow rule and attributes the nonlinearity / path-count hotspots to source functions, classifying +each as **cut** (a function of the contract under test → a generator's over-approx/precise model), **library** +(inlined lib → library model), **external** (linked dependency → dependency model), or **cvl-model** +(already summarized). The CUT and the scene's linked contracts are read from the run's treeViewStatus, +so nothing is protocol-specific. + + python -m summarization_detector.difficulty_profile [more jobs...] --min-minutes 20 + python -m summarization_detector.difficulty_profile --json # for pipeline consumption + +Use it to decide, with evidence, whether a scene's timeouts are in dependency/library code (a dependency/library model +dependency/library model helps) or in the contract under test itself (a per-function model / harness). diff --git a/summarization_detector/__init__.py b/summarization_detector/__init__.py new file mode 100644 index 00000000..f5590083 --- /dev/null +++ b/summarization_detector/__init__.py @@ -0,0 +1,63 @@ +"""Summarization-target detector — a standalone AutoProver tool. + +From a prover run it ranks the functions worth summarizing: a candidate list with WHY each is +prover-hostile (the hard-op signal / category), WHERE it can be summarized (caller boundaries), and, for a +curated public-library match, a suggested summary. It suggests WHAT to summarize — not the summarization +strategy (per-function summary, whole-contract symbolic model, …) and not how the summary is written; the +CVL-generation / invariant agent (or smtool) does that. It fetches the prover's output via +`prover_output_utility` and parses the solc AST dump (from `certoraRun --dump_asts`) with +`certora_autosetup.solidity_ast`. Invoke via `detect()` or the CLI +(`python -m summarization_detector` / `detect-summaries`). +""" +from .detect import ( + Boundary, + Candidate, + DetectionReport, + HashSignal, + HostileCategory, + CuratedEntry, + HostileMatch, + detect, + detect_from, + scan_ast, + classify_hostile, + surviving_hostile, + reachable_from_main, + cone_weights, +) +from .sources import detect_url, cut_from_conf, find_run_conf, fetch_surviving_graphs +from .difficulty_profile import ( + ProfileReport, + SlowRule, + Hotspot, + profile_job, + profile_jobs, + aggregate_by_class, +) + +__all__ = [ + "Boundary", + "Candidate", + "DetectionReport", + "HashSignal", + "detect", + "detect_from", + "detect_url", + "scan_ast", + "reachable_from_main", + "cone_weights", + "cut_from_conf", + "find_run_conf", + "ProfileReport", + "SlowRule", + "Hotspot", + "profile_job", + "profile_jobs", + "aggregate_by_class", + "HostileCategory", + "CuratedEntry", + "HostileMatch", + "classify_hostile", + "surviving_hostile", + "fetch_surviving_graphs", +] diff --git a/summarization_detector/__main__.py b/summarization_detector/__main__.py new file mode 100644 index 00000000..eb53e2f3 --- /dev/null +++ b/summarization_detector/__main__.py @@ -0,0 +1,3 @@ +from .cli import main + +raise SystemExit(main()) diff --git a/summarization_detector/cli.py b/summarization_detector/cli.py new file mode 100644 index 00000000..81bd37da --- /dev/null +++ b/summarization_detector/cli.py @@ -0,0 +1,55 @@ +"""Standalone CLI for the summarization-target detector — `python -m summarization_detector` or the +installed `detect-summaries` command. + +The one input is `--url` (a prover-run URL): it fetches the sources + conf, derives the main contract, +generates the AST, and pulls the difficulty report — getting as much signal as available. The lower-level +flags (`--ast`/`--conf`/`--cut`/…) are overrides / an offline path when you already have the artifacts.""" +import argparse +import json + +from .detect import detect +from .sources import detect_url + + +def main(argv: list[str] | None = None) -> int: + p = argparse.ArgumentParser( + prog="detect-summaries", + description="Rank the functions worth summarizing in a scene (and how: over-approx vs symbolic " + "model). Give just --url; everything else is derived.") + p.add_argument("--url", default=None, + help="prover-run URL — the only input needed: fetches sources + conf, derives the main " + "contract, generates the AST, and pulls the difficulty report.") + p.add_argument("--cut", default=None, + help="override the main/parametric contract (else derived from the conf's `verify`).") + p.add_argument("--work-dir", default=None, + help="where to fetch sources + generate the AST (default: a temp dir). --url mode only.") + p.add_argument("--external-call-graph", default=None, + help="the prover's externalCallGraph.json — enables the reachability-from-CUT gate " + "(auto-found in the fetched tree when present).") + p.add_argument("--solc-dir", default=None, + help="directory prepended to PATH so the conf's solcN.NN resolves.") + p.add_argument("--include-dependencies", action="store_true", + help="also report hashing in lib/ dependency code (off by default).") + p.add_argument("--json", action="store_true", + help="emit the report as JSON (for pipeline/tool consumption) instead of text.") + # offline / override path (when you already have the artifacts instead of a URL): + p.add_argument("--ast", default=None, help="path to a solc AST dump (.asts.json).") + p.add_argument("--conf", default=None, help="a .conf to generate the AST from (offline, needs --cut).") + p.add_argument("--job-url", default=None, + help="difficulty-report URL for the offline path (with --ast/--conf).") + a = p.parse_args(argv) + + if a.url: + report = detect_url(a.url, work_dir=a.work_dir, solc_dir=a.solc_dir, cut=a.cut, + external_call_graph=a.external_call_graph, + include_dependencies=a.include_dependencies) + else: + if not (a.ast or a.conf): + p.error("give --url, or the offline path: --ast/--conf (+ --cut).") + if not a.cut: + p.error("--cut is required on the offline path (no --url to derive it from).") + report = detect(a.job_url, ast_path=a.ast, conf=a.conf, cut=a.cut, solc_dir=a.solc_dir, + external_call_graph=a.external_call_graph, + include_dependencies=a.include_dependencies) + print(json.dumps(report.to_dict(), indent=2) if a.json else report.format()) + return 0 diff --git a/summarization_detector/detect.py b/summarization_detector/detect.py new file mode 100644 index 00000000..e5012961 --- /dev/null +++ b/summarization_detector/detect.py @@ -0,0 +1,1229 @@ +"""Summarization-target DETECTOR — from ONE prover run, rank the functions worth summarizing (and, for a +curated match, suggest how). + +Run this on a SANITY run (a `satisfy true` per-method reachability check) to decide what to +summarize BEFORE the real rules exist; a downstream generator then produces the summaries. + +A sanity run processes every method through the full build+optimize TAC pipeline, so the expensive paths +(nonlinear, bitwise, hashing) stay IN the program and appear in the difficulty/live statistics and the +surviving call graph. Only the SMT SOLVE is trivial — it may exit a method via one easy path without +satisfying the hard nonlinear constraints — so solve TIME is not a cost signal, but the TAC-derived +difficulty statistics ARE (they reflect the code in the problem, not the path the solver took). + +FOUR signals, each catching a different cost class: + + 1. NONLINEAR (SMT phase) — the prover's own difficulty report (`difficulty.fetch_difficulty`): ranked + functions whose INLINED body contributes nonlinear ops (mulDiv, pow, sqrt, div). Catches the + timeouts that reach SMT. + 2. HASHING / ENCODING — a Solidity-AST walk (`scan_ast`): calls to keccak256 / sha256 / ecrecover / + abi.encode* in a function body. These choke the BUILD / points-to phase, so they produce NO + nonlinear hotspot — invisible to signal 1 (the getConditionId case). We read the AST (never regex + the .sol) so comments and `assembly {}` (storage-slot keccaks) are excluded for free — a Yul call is + not a Solidity `FunctionCall` node. + 3. RESOLVED-EXPENSIVE EXTERNAL — a signal-1 hotspot whose owning contract is NOT the CUT: a call that + ALREADY resolved/linked to a real contract/oracle but is expensive once inlined. We do NOT touch + UNRESOLVED (`[?]`) calls — resolving those is call-resolution's job (a separate tool). + 4. SURVIVING HOSTILE PRIMITIVE — from the postOptimize SurvivingCallGraph dumps (`surviving_hostile`): + a RAW (non-ghost) prover-hostile primitive — bitwise scan / in-memory sort / symbolic exp / mulDiv, + recognized by a generic operation catalog + a curated public-library overlay — that survives + optimization, carrying which entry methods reach it and a candidate summary. Catches bitwise/sort, + which contribute no nonlinear op and so are invisible to signal 1. + +AST acquisition (`ensure_ast`): the raw solc AST (`.asts.json`) is all we need — its `FunctionDefinition` +nodes carry name + `stateMutability` + `visibility`, so no separate methods manifest is needed. Pass an +existing `ast_path` (from a prior `--dump_asts` run), or +pass a `conf` and we run `certoraRun --compilation_steps_only --dump_asts` ourselves (standalone mode). + +Pure cores (`scan_ast`, `detect_from`) take already-fetched inputs so they are unit-testable offline; +`detect` is the thin orchestrator; `cli.main` is the standalone command. It decides WHAT to summarize; a +downstream generator produces the summaries. It only REUSES AutoProver code — the local `difficulty` +module for the difficulty signal and `certora_autosetup.solidity_ast.stream_raw_units` for the AST. +""" +import json +import re +import subprocess +import sys +from dataclasses import asdict, dataclass, field +from pathlib import Path + +from certora_autosetup.solidity_ast import stream_raw_units + +from .difficulty import DifficultyReport, fetch_difficulty + +# ---------------------------------------------------------------- signal 2: AST hashing/encoding calls +# The TRIGGER is an actual hash builtin — a global `Identifier` callee. Yul (assembly) calls are +# `YulFunctionCall`, never Solidity `FunctionCall`, so storage-slot keccaks are excluded automatically. +_HASH_IDENT = {"keccak256", "sha256", "ripemd160", "ecrecover"} +# `abi.encode` / `abi.encodePacked` (MemberAccess on the `abi` object) are CONTEXT — what is being hashed +# (they decide the length class), never a trigger on their own. Deliberately EXCLUDES encodeWithSelector / +# encodeCall / encodeWithSignature: those build calldata for an external CALL, not a hash. +_ENCODERS = {"encode", "encodePacked"} + + +@dataclass +class HashSignal: + """Signal-2 record: a function whose body performs semantic hashing/encoding (build/points-to cost + the SMT difficulty report cannot see). Mutability/visibility come from the same FunctionDefinition + node — the classifier's pure/view test needs no separate source.""" + function: str # "Contract.fn" ("fn" for a file-level free function) + contract: str # owning contract/library ("" for a free function) + name: str + mutability: str # "pure" | "view" | "payable" | "nonpayable" | "" + visibility: str # "external" | "public" | "internal" | "private" + patterns: tuple[str, ...] # the hashing/encoding calls found, e.g. ("keccak256", "abi.encodePacked") + file: str # absolute source path + is_dependency: bool # under a lib/ dependency tree (deprioritize vs project src/) + dynamic_input: bool = False # hashes UNBOUNDED-length data (bytes/string/dynamic array) — the costly + # kind (the prover models up to hashing_length_bound bytes). A hash over + # only fixed-size fields (the typical EIP-712 digest) is bounded/cheap. + + +# Output caps — keep the report (and the prompt it renders into) bounded. The score is only comparable +# WITHIN a signal category (nonlinear = % of the rule's nonlinear ops; surviving = flat; hashing = fixed), +# so each category is capped on its own rather than by a single cross-category top-N (which would let the +# nonlinear %s crowd out the surviving/hashing targets). +MAX_PER_CATEGORY = {"primitive": 10, "nonlinear": 10, "hashing": 10} +NONLINEAR_MIN_PCT = 15 # drop difficulty hotspots below this % of the rule's nonlinear ops (long tail) + +# Flat score for a surviving hostile primitive (signal 4). It does NOT scale with how many entry methods +# reach it: reach is breadth, not summarization priority, so a primitive reached by 2 methods ranks with one +# reached by 20. The reach count rides along as context only (`reaching_count`). +SURVIVING_SCORE = 50.0 + + +@dataclass +class Boundary: + """A caller (or, for a "down" target, a shared inlined primitive) offered as an alternative place to + summarize the leaf. Only EXPRESSIBLE, INTERNAL boundaries are kept — has a return AND all param/return + types are CVL-clean (no void, opaque dynamic bytes/string, mapping, or function type), and not a + public/external entry point (the rule's subject, never a summarization site) — so the signature alone + conveys feasibility. `mutating` (writes state, NOT view/pure — summarizing it as a value would erase + side effects the properties may observe) is the one remaining caveat. `hops` = call-graph distance from + the candidate. `direction` = "up" (a CALLER to summarize at — the hashing-leaf case) or "down" (a shared + nonlinear PRIMITIVE the candidate inlines — the real arg-based target). `shared` = for a "down" target, + how many sibling candidates also inline it (fan-in). Best: non-mutating, nearest.""" + function: str + hops: int + signature: str + mutating: bool + direction: str = "up" + shared: int = 1 + + +@dataclass +class Candidate: + """A function worth summarizing, with WHY (`signals`) and, for a curated match, HOW (`candidate_summary`).""" + function: str # "Contract.fn" (the contract prefix is part of the identifier) + signals: tuple[str, ...] # why flagged: "nonlinear" (difficulty %), "hashing" (AST), "external" + # (cross-contract modifier), or a catalog hard-op class naming the exact + # primitive: "in-memory-sort" / "bitwise-scan" / "symbolic-exp" / + # "nonlinear-mulDiv". No liveness label — every candidate reaches SMT. + score: float # rank key (higher = summarize first) + evidence: str # human-readable justification + file: str = "" # source file of the function ("" if unresolved) + line: int | None = None # 1-based start line of the function (None if unresolved) + signature: str = "" # readable `fn(paramTypes) -> returnTypes` (from the AST; "" if unresolved) + mutating: bool | None = None # True if state-changing (not view/pure); None if unresolved from the AST + reaching_count: int = 0 # how many entry methods' postOptimize TAC keeps this primitive (breadth) + summarizable: bool = True # the prover's own `summarizable` flag (surviving graph) + candidate_summary: str = "" # suggested summary (curated EXACT or generic over-approx) + boundaries: list[Boundary] = field(default_factory=list) # caller boundaries to summarize at instead + + +@dataclass +class DetectionReport: + candidates: list[Candidate] = field(default_factory=list) + dropped: int = 0 # candidates cut by the per-category caps (0 = the whole ranked list is present) + + def is_empty(self) -> bool: + return not self.candidates + + def to_dict(self) -> dict: + """Machine-readable form for a consuming pipeline: each candidate (the problematic function, why, + and rank) with its caller-boundary shortlist. Fields left at their default (unresolved location, + no reach, no summary, no boundaries, summarizable) are OMITTED — the consumer assumes the default, + and the prompt this renders into stays lean. The emitted shape is the `schema.py` TypedDicts + (`HostileCandidate` / `HostileBoundary`) — keep those in step with this and the `Candidate` fields.""" + defaults = {"file": "", "line": None, "signature": "", "mutating": None, "reaching_count": 0, + "summarizable": True, "candidate_summary": "", "boundaries": []} + candidates = [] + for c in self.candidates: + d = {k: v for k, v in asdict(c).items() if defaults.get(k, object()) != v} + candidates.append(d) + return {"candidates": candidates, "dropped": self.dropped} + + def format(self) -> str: + if self.is_empty(): + return "no summarization candidates detected" + out = ["summarization candidates (what to summarize first, and how):"] + for c in self.candidates: + reach = f" reaches {c.reaching_count}" if c.reaching_count else "" + loc = f" {c.file}:{c.line}" if c.file else "" + out.append(f" {c.score:5.1f} {c.function} <{','.join(c.signals)}>{reach}{loc}") + out.append(f" {c.evidence}") + if c.candidate_summary: + out.append(f" candidate: {c.candidate_summary}") + for b in c.boundaries: + # only expressible, internal boundaries survive the filter, so mutating is the last caveat + feas = "state-changing — prefer a pure boundary" if b.mutating else "summarizable here" + if b.direction == "down": # a shared nonlinear primitive (descend) + arrow = "↓" + tag = f"shared ×{b.shared}, {feas}" if b.shared > 1 else feas + else: # a caller boundary (walk up) + arrow, tag = "↑", feas + out.append(f" {arrow} +{b.hops} {b.signature} [{tag}]") + return "\n".join(out) + + +# ---------------------------------------------------------------- signal 2 core: the AST walk +def _span(node: dict) -> tuple[int, int] | None: + """The node's `src` = "offset:length:fileId" -> (start, end) byte offsets. All nodes within one + source file share a fileId, so containment by (start, end) alone attributes a call to its function.""" + src = node.get("src") + if not isinstance(src, str): + return None + try: + off, length, _fid = src.split(":") + return int(off), int(off) + int(length) + except ValueError: + return None + + +def _classify_call(call: dict) -> tuple[str, str] | None: + """Classify a FunctionCall as a hash TRIGGER or an ENCODER (context), or None. Returns ("hash", + "keccak256") for a hash builtin, ("encode", "abi.encodePacked") for an abi encoder (base guarded so a + user type's own `.encode()` is not mistaken for `abi.encode`). Only a "hash" makes a function a + candidate; "encode" merely informs the length class.""" + ex = call.get("expression") or {} + nt = ex.get("nodeType") + if nt == "Identifier" and ex.get("name") in _HASH_IDENT: + return ("hash", ex["name"]) + if nt == "MemberAccess" and ex.get("memberName") in _ENCODERS: + base = ex.get("expression") or {} + if base.get("nodeType") == "Identifier" and base.get("name") == "abi": + return ("encode", "abi." + ex["memberName"]) + return None + + +def _arg_types(call: dict) -> tuple[str, ...]: + """The solc typeStrings of a call's arguments (e.g. ('address','bytes32','uint256')).""" + out = [] + for a in call.get("arguments") or []: + if isinstance(a, dict): + out.append((a.get("typeDescriptions") or {}).get("typeString") or "") + return tuple(out) + + +def _is_dynamic_type(ts: str) -> bool: + """True if a solc typeString denotes UNBOUNDED-length data: `bytes`, `string`, or a dynamic array + `T[]`. Fixed-width (`bytes32`, `uint256`, `address`, a fixed array `T[5]`, a struct) is False — + the storage location suffix (` memory`/` calldata`/` storage`) is dropped first.""" + if not ts: + return False + base = ts.split(" ", 1)[0] # "uint256[] memory" -> "uint256[]"; "bytes memory" -> "bytes" + if base.endswith("[]"): + return True + if base in ("bytes", "string"): + return True + return False + + +def _refs(node) -> set: + """All positive `referencedDeclaration` ids in an expression subtree — the declarations it reads.""" + out: set = set() + if isinstance(node, dict): + rd = node.get("referencedDeclaration") + if isinstance(rd, int) and rd > 0: + out.add(rd) + for v in node.values(): + out |= _refs(v) + elif isinstance(node, list): + for v in node: + out |= _refs(v) + return out + + +def _dynamic_input(sites: list, param_ids: set) -> bool: + """Whether a function's hashing consumes unbounded-length USER data. A site counts only when it has a + dynamic-typed operand AND the call references a function PARAMETER (`param_ids`) — so a hash over a + constant/immutable/literal (e.g. `keccak256(bytes(name))` for a fixed EIP-712 name) is NOT dynamic + despite the `bytes` type. `sites` are (pattern, arg_types, refs) per hash/encode call. An `abi.encode*` + with a dynamic arg is the source; a bare `keccak256(x)` counts only when no encoder feeds it (else its + `bytes memory` arg is just the encode result, whose class the encode site already decided).""" + has_encode = any(p.startswith("abi.") for p, _, _ in sites) + for pattern, args, refs in sites: + if not (refs & param_ids): # not user-controlled -> not the expensive case + continue + if pattern.startswith("abi."): + if any(_is_dynamic_type(t) for t in args): + return True + elif not has_encode and any(_is_dynamic_type(t) for t in args): + return True + return False + + +def _tightest(off: int, spans: list) -> tuple | None: + """The smallest span in `spans` (each `((start,end), *payload)`) that contains offset `off` — the + innermost enclosing function/contract.""" + best = None + for entry in spans: + (s, e) = entry[0] + if s <= off < e and (best is None or (e - s) < (best[0][1] - best[0][0])): + best = entry + return best + + +def scan_ast(ast_path: str | Path) -> list[HashSignal]: + """Signal 2: walk the raw solc AST dump (`.asts.json`) and return every function that performs + semantic hashing/encoding. Streams the file (it is often hundreds of MB) and processes each source + file ONCE — a file recurs under every compilation unit that imports it, with re-numbered node ids but + identical content, so we dedup by absolute path. Nodes are pre-flattened (every descendant is a + top-level entry), so a single pass over each file's nodes finds all FunctionCall/FunctionDefinition/ + ContractDefinition nodes; attribution is by src-offset containment.""" + out: dict[str, HashSignal] = {} + sites: dict[str, list] = {} # fnkey -> [(pattern, arg_types, refs)...] for the dynamic/fixed decision + hash_pats: dict[str, set] = {} # fnkey -> the set of HASH-builtin names it calls (for ecrecover-only) + param_ids: dict[str, set] = {} # fnkey -> its parameter declaration ids (for the user-input test) + seen: set[str] = set() + for _rel, pdata in stream_raw_units(Path(ast_path)): + if not isinstance(pdata, dict): + continue + for absp, nodes in pdata.items(): + if absp in seen or not isinstance(nodes, dict): + continue + seen.add(absp) + contracts: list = [] # ((start,end), name) + funcs: list = [] # ((start,end), name, mutability, visibility, param_ids) + hits: list = [] # (offset, kind, pattern, arg_types, refs) + for node in nodes.values(): + if not isinstance(node, dict): + continue + sp = _span(node) + if sp is None: + continue + nt = node.get("nodeType") + if nt == "ContractDefinition": + contracts.append((sp, node.get("name") or "")) + elif nt == "FunctionDefinition": + pids = {p.get("id") for p in ((node.get("parameters") or {}).get("parameters") or []) + if isinstance(p, dict) and isinstance(p.get("id"), int)} + funcs.append((sp, node.get("name") or "", node.get("stateMutability") or "", + node.get("visibility") or "", pids)) + elif nt == "FunctionCall": + kp = _classify_call(node) + if kp: + hits.append((sp[0], kp[0], kp[1], _arg_types(node), _refs(node.get("arguments")))) + # AST source keys are project-relative (e.g. "lib/solady/src/tokens/ERC20.sol"), so test for a + # dependency-root path COMPONENT rather than a "/lib/" substring (which misses a leading "lib/"). + parts = set(Path(absp).parts) + is_dep = bool(parts & {"lib", "node_modules", ".certora_internal"}) + per_fn: dict[str, tuple] = {} # fnkey -> (contract, name, mut, vis) + for off, kind, pat, argtypes, refs in hits: + f = _tightest(off, funcs) + if f is None: # a call at file scope — no owning fn + continue + (_fsp, fname, mut, vis, pids) = f + c = _tightest(f[0][0], contracts) + cname = c[1] if c else "" + key = f"{cname}.{fname}" if cname else fname + per_fn[key] = (cname, fname, mut, vis) + sites.setdefault(key, []).append((pat, argtypes, refs)) + param_ids[key] = pids + if kind == "hash": + hash_pats.setdefault(key, set()).add(pat) + for key, (cname, fname, mut, vis) in per_fn.items(): + if not hash_pats.get(key): # encoder-only (calldata/serialization) — not hashing + continue + pats = tuple(sorted({p for p, _, _ in sites[key]})) + rec = out.get(key) + if rec is None: + out[key] = HashSignal(function=key, contract=cname, name=fname, mutability=mut, + visibility=vis, patterns=pats, file=absp, is_dependency=is_dep) + else: + rec.patterns = tuple(sorted({*rec.patterns, *pats})) + # ecrecover ONLY (signature recovery) is not usefully over-approximable — its result is a recovered + # address with no sound property tighter than havoc. Drop those (a function that also hashes stays). + out = {k: v for k, v in out.items() if not (hash_pats.get(k, set()) <= {"ecrecover"})} + for key, rec in out.items(): + rec.dynamic_input = _dynamic_input(sites.get(key, []), param_ids.get(key, set())) + return sorted(out.values(), key=lambda h: h.function) + + +def _function_locations(ast_path: str | Path, + sources_root: str | Path | None = None) -> dict[str, tuple[str, int | None]]: + """`Contract.fn` -> (source_file, 1-based start line | None). The FILE is the AST node group's path. + The LINE resolves the FunctionDefinition byte-offset against the source when it is readable under + `sources_root` (project root the AST paths are relative to); None when no `sources_root` or the file + can't be read. Streams the AST once, deduping repeated source files by path (as `scan_ast` does).""" + import bisect + out: dict[str, tuple[str, int | None]] = {} + newlines: dict[str, list[int] | None] = {} # file -> newline byte offsets (None = unreadable) + + def _relativize(p: str) -> str: + # solc records source-unit keys as a MIX of project-relative and absolute paths (depending on how + # each was imported/remapped); normalize to project-relative so every entry's `file` is uniform. + if sources_root is None: + return p + try: + return str(Path(p).relative_to(Path(sources_root))) + except ValueError: + return p # already relative / not under the root — leave as-is + + def _line_of(file: str, offset: int) -> int | None: + if sources_root is None: + return None + if file not in newlines: + try: + data = (Path(sources_root) / file).read_bytes() + newlines[file] = [i for i, b in enumerate(data) if b == 0x0A] + except Exception: + newlines[file] = None + nl = newlines[file] + return None if nl is None else bisect.bisect_right(nl, offset) + 1 + + seen: set[str] = set() + for _rel, pdata in stream_raw_units(Path(ast_path)): + if not isinstance(pdata, dict): + continue + for absp, nodes in pdata.items(): + if absp in seen or not isinstance(nodes, dict): + continue + seen.add(absp) + contracts: list = [] + funcs: list = [] + for node in nodes.values(): + if not isinstance(node, dict): + continue + sp = _span(node) + if sp is None: + continue + nt = node.get("nodeType") + if nt == "ContractDefinition": + contracts.append((sp, node.get("name") or "")) + elif nt == "FunctionDefinition" and node.get("name"): + funcs.append((sp, node["name"])) + for fsp, fname in funcs: + c = _tightest(fsp[0], contracts) + cname = c[1] if c else "" + qual = f"{cname}.{fname}" if cname else fname + out[qual] = (_relativize(absp), _line_of(absp, fsp[0])) # read line from absp, store relative + return out + + +# ---------------------------------------------------------------- signal 4: surviving hostile primitives +# From a sanity run's postOptimize SurvivingCallGraph dumps (one per entry method): the functions still in +# the optimized TAC. A RAW (non-ghost) prover-hostile primitive that survives is a summarization candidate; +# an already-applied summary appears as a `CVL/Ghost Function` stand-in (excluded — it IS the summary). +# GENERIC_RULES recognize the hostile OPERATION by conventional primitive-name tokens (protocol-agnostic); +# a CURATED overlay maps specific PUBLIC libraries to their known EXACT summaries. +@dataclass(frozen=True) +class HostileCategory: + key: str + match: "re.Pattern[str]" # generic operation-name pattern + reason: str # why it is prover-hostile + + +# Linear constant-scaling conversions (× a compile-time constant) are cheap — exclude so a fixed-point +# name doesn't misfire (e.g. `bpsToWad` = value·1e‹k›). +_LINEAR_SCALE = re.compile( + r"\b(bpsToWad|bpsToRay|toWad|toRay|wadToRay|rayToWad|fromWad(Down|Up)?|fromRay(Down|Up)?|" + r"fromBps(Down|Up)?|scaleBy|normalizeDecimals)\b", re.I) + +# GENERIC operation categories — matched on conventional primitive-name tokens, no project names. A generic +# match reports WHAT (category + why) only; it suggests NO summary — the agent writes the one that is sound +# for the property at hand. +GENERIC_RULES: tuple[HostileCategory, ...] = ( + HostileCategory( + key="bitwise-scan", + match=re.compile(r"(?:\b|_)(fls|flz|clz|ctz|msb|lsb)\b|pop[_]?count|findLastSet|findFirstSet|bitLen", + re.I), + reason="word-wide bit scan / population count — under-specified bitwise ops; a major imprecision " + "and timeout source when inlined", + ), + HostileCategory( + key="in-memory-sort", + match=re.compile(r"(?:\b|_)(sort|quickSort|mergeSort|heapSort|insertionSort)(ByKey|Asc\w*|Desc\w*)?\b", + re.I), + reason="in-place permutation sort over an in-memory list — unrolls into the loop bound; expensive", + ), + HostileCategory( + key="symbolic-exp", + match=re.compile(r"(?:\b|_)(exp|pow|rpow|power)(?=[_A-Z]|\b)|(?<=[a-z])(Exp|Pow|Rpow|Power)(?=[_A-Z(]|\b)"), + reason="exponentiation with a symbolic exponent — an unrolled loop; prover-hostile", + ), + HostileCategory( + key="nonlinear-mulDiv", + match=re.compile(r"(?:\b|_)(mulDiv\w*|fullMulDiv\w*|mulWad\w*|divWad\w*|rayMul\w*|rayDiv\w*|" + r"wadMul\w*|wadDiv\w*|percentMul\w*|percentDiv\w*)\b", re.I), + reason="256/512-bit multiply-divide of two symbolic operands — nonlinear SMT", + ), +) + +# The catalog categories double as SIGNALS: a catalog-matched candidate carries its category +# (`in-memory-sort`, `bitwise-scan`, …) directly in `signals` rather than a generic `primitive` + a +# separate `category` field. This set is how the per-category cap groups them all under "primitive". +_PRIMITIVE_CATEGORIES = frozenset(c.key for c in GENERIC_RULES) + + +@dataclass(frozen=True) +class CuratedEntry: + match: "re.Pattern[str]" # a specific "Contract.func(sig)" pattern for a known PUBLIC library + category: str # one of the GENERIC_RULES keys + summary: str # the concrete summary text, INCLUDING any soundness caveat — a curated + # entry may be exact, or a documented over-/under-approximation + note: str = "" + + +# CURATED overlay — specific PUBLIC libraries → a known-good, concrete summary. Only public, widely-used +# third-party libraries belong here; a protocol's own private math is caught by the GENERIC rules above and +# carries no suggested summary. +CURATED_SUMMARIES: tuple[CuratedEntry, ...] = ( + CuratedEntry( + match=re.compile(r"\b(WadRayMath|PercentageMath)\.(ray|wad|percent)", re.I), + category="nonlinear-mulDiv", + summary="=> WAD/RAY fixed-point mulDiv summary (EXACT floor/ceil of x·y/scale).", + ), + CuratedEntry( + match=re.compile(r"\b(Math|MathUpgradeable)\.mulDiv\b"), + category="nonlinear-mulDiv", + summary="=> OZ_Math.mulDiv curated summary (EXACT).", + ), + CuratedEntry( + match=re.compile(r"\bFixedPointMathLib\.|\bFullMath\.mulDiv\b|\bPRBMath"), + category="nonlinear-mulDiv", + summary="=> FixedPointMathLib / FullMath / PRB curated summary (EXACT).", + ), +) + + +@dataclass(frozen=True) +class HostileMatch: + category: str + reason: str + candidate_summary: str # a concrete summary from the curated overlay, or "" for a generic match + curated: bool + + +def classify_hostile(name: str) -> HostileMatch | None: + """Resolve a solidity function name to a hostile match, or None. A GENERIC operation rule fires first + (protocol-agnostic) and reports only WHAT (category + why); a CURATED overlay entry, if any, attaches a + concrete summary. Linear constant-scaling conversions are excluded up front.""" + if _LINEAR_SCALE.search(name): + return None + cat = next((c for c in GENERIC_RULES if c.match.search(name)), None) + if cat is None: + return None + cur = next((e for e in CURATED_SUMMARIES if e.match.search(name)), None) + if cur is not None: + return HostileMatch(cat.key, cat.reason, candidate_summary=cur.summary, curated=True) + return HostileMatch(cat.key, cat.reason, candidate_summary="", curated=False) + + +def _entry_of_rule(rule: str) -> str: + """The entry-method name from a sanity rule name. Each method yields TWO surviving graphs — the + `-Satisfy_sanity_check_failed_...` (reachability) and the `-Assertions` (assertion) run — so strip + either suffix to the bare ``; both then dedup to one entry (`sanity--Satisfy...` and + `sanity--Assertions` -> ``).""" + m = re.match(r"sanity-(?P.+?)-(?:Satisfy|Assertions)", rule or "") + return m.group("sig") if m else (rule or "") + + +def _surviving_names(graph: dict) -> list[tuple[str, bool]]: + """(function_name, summarizable) for every procedure/internal function in one SurvivingCallGraph.""" + names: list[tuple[str, bool]] = [] + for p in graph.get("procedures", []) or []: + if p.get("procId"): + names.append((p["procId"], True)) + for f in graph.get("internalFunctions", []) or []: + nm = f.get("name") or f.get("procId") + if nm: + names.append((nm, bool(f.get("summarizable", True)))) + return names + + +def surviving_hostile(graphs: list[dict]) -> dict[str, dict]: + """Aggregate raw (non-ghost) hostile primitives across postOptimize SurvivingCallGraph dumps. Returns + function -> {category, reason, reaching_methods, summarizable, candidate_summary} (candidate_summary is + "" for a generic match, the curated text for a curated one). A `CVL/Ghost` survivor is an + already-applied summary and is skipped.""" + out: dict[str, dict] = {} + for g in graphs: + if (g.get("phase") or "").lower() not in ("postoptimize", ""): + continue + entry = _entry_of_rule(g.get("rule", "")) + for name, summarizable in _surviving_names(g): + if _is_cvl_ghost(name): + continue + m = classify_hostile(name) + if m is None: + continue + rec = out.get(name) + if rec is None: + rec = out[name] = {"category": m.category, "reason": m.reason, "reaching_methods": [], + "summarizable": True, "candidate_summary": m.candidate_summary} + if entry not in rec["reaching_methods"]: + rec["reaching_methods"].append(entry) + rec["summarizable"] = rec["summarizable"] and summarizable + return out + + +# ---------------------------------------------------------------- AST acquisition (optional-arg design) +_MISSING_IMPORT_RE = re.compile(r'\d+:\d+:"([^"]+)"') + + +def _touch_missing_imported_specs(*outputs: str) -> list: + """Recreate empty placeholders for imports certoraRun reports as missing. The prover prints + `... import declarations do not import existing .spec files:` then `::""` entries. + The real files were skipped on upload precisely because they are EMPTY, so an empty placeholder is + faithful. Returns the paths created (empty list if the failure is something else).""" + created: list = [] + for out in outputs: + if "do not import existing" not in out: + continue + for m in _MISSING_IMPORT_RE.finditer(out): + p = Path("".join(m.group(1).split())) # certoraRun wraps long paths across lines in captured + if not p.exists(): # (non-tty) output — rejoin before treating as a path + p.parent.mkdir(parents=True, exist_ok=True) + p.write_text("") + created.append(p) + return created + + +def ensure_ast(ast_path: str | Path | None = None, *, conf: str | Path | None = None, + solc_dir: str | Path | None = None) -> Path: + """Return a path to a solc AST dump. If `ast_path` is given (from a prior `--dump_asts` run, or a + prior run), use it. Otherwise run `certoraRun --compilation_steps_only --dump_asts` (standalone + mode) and return the freshest `.asts.json` it writes under `.certora_internal/`. `solc_dir` is + prepended to PATH so the conf's `solcN.NN` resolves. Raises if neither input suffices.""" + if ast_path is not None: + p = Path(ast_path) + if not p.exists(): + raise FileNotFoundError(f"ast_path does not exist: {p}") + return p + if conf is None: + raise ValueError("provide ast_path (existing AST) or conf (to generate one via certoraRun)") + conf = Path(conf) + work = conf.parent + env_path = None + if solc_dir is not None: + import os + env_path = {**os.environ, "PATH": f"{solc_dir}:{os.environ.get('PATH', '')}"} + cmd = ["certoraRun", conf.name, "--compilation_steps_only", "--dump_asts"] + proc = subprocess.run(cmd, cwd=work, env=env_path, capture_output=True, text=True) + # TEMPORARY WORKAROUND: the source-files upload skips EMPTY files, so an empty importable source that + # another file imports is dropped while its importer is kept — the fetched scene then fails to compile + # on the missing import (fixed upstream for NEW runs; existing runs stay broken). The missing file is + # empty by definition, so recreate the missing imported spec(s) as empty placeholders and retry. + for _ in range(5): + if proc.returncode == 0: + break + created = _touch_missing_imported_specs(proc.stdout or "", proc.stderr or "") + if not created: + break + print(f"[detect] created {len(created)} empty placeholder spec(s) for skipped-empty imports; retrying", + file=sys.stderr) + proc = subprocess.run(cmd, cwd=work, env=env_path, capture_output=True, text=True) + if proc.returncode != 0: + tail = "\n".join((proc.stderr or proc.stdout or "").strip().splitlines()[-25:]) + raise RuntimeError( + f"certoraRun AST generation failed (exit {proc.returncode}) in {work} for {conf.name}:\n{tail}\n" + f"(hint: if the error is a missing solc, pass solc_dir / --solc-dir so the conf's solcN.NN " + f"resolves; if it is a missing source/package, the fetched scene may be incomplete.)") + dumps = sorted((work / ".certora_internal").rglob("*.asts.json"), key=lambda p: p.stat().st_mtime) + if not dumps: + raise FileNotFoundError(f"certoraRun produced no .asts.json under {work}/.certora_internal") + return dumps[-1] + + +# ---------------------------------------------------------------- fusion + classifier +def _contract_of(procid: str) -> str: + """Contract/library prefix of a difficulty procId ('SomeLib.fn' -> 'SomeLib'; bare -> '').""" + return procid.split(".", 1)[0] if "." in procid else "" + + +def _is_cvl_ghost(function: str) -> bool: + """A `CVL/Ghost Function '...'` hotspot is an ALREADY-APPLIED CVL summary / ghost, not a Solidity + function to summarize — so it is never a candidate (it IS the summary). The prover labels these + procIds distinctly; real Solidity hotspots are `Contract.fn` with a source location.""" + return function.lstrip().startswith(("CVL/Ghost", "CVL Function", "Ghost")) + + +def _strip_procid(function: str) -> str: + """Normalize a prover procId to `Contract.fn` for the detector: drop a leading `(internal)` / + `(external)` marker the prover prefixes to inlined-function hotspots. Without this the marker leaks + into the contract parse, so the CUT's OWN internal function (`(internal) Stonks.foo`) is misread as a + different contract and wrongly classified as a resolved-external. (The raw marker stays intact in the + shared DifficultyReport, which a downstream refine step may reuse.)""" + return re.sub(r"^\((?:internal|external)\)\s*", "", function.strip()) + + +def _parse_location(loc: str) -> tuple[str, int | None]: + """Split a difficulty hotspot location (`file:line`, or just `file`) into `(file, line)`. `line` is + `None` when it is absent or non-numeric. Source paths carry no `:`, so the last segment is the line.""" + file, _, line = loc.rpartition(":") + if file and line.isdigit(): + return file, int(line) + return loc, None + + +def _bucket(c: Candidate) -> str: + """The cap bucket a candidate falls in (its score is only comparable within it). Any catalog hard-op + signal -> "primitive"; else nonlinear -> "nonlinear"; else "hashing". `external` is a modifier, not a + bucket of its own.""" + if any(s in _PRIMITIVE_CATEGORIES for s in c.signals): + return "primitive" + if "nonlinear" in c.signals: + return "nonlinear" + return "hashing" + + +def detect_from(hash_signals: list[HashSignal], difficulty: DifficultyReport, *, cut: str, + include_dependencies: bool = False, + cone_weight: dict[str, int] | None = None, + surviving: dict[str, dict] | None = None) -> DetectionReport: + """Fuse the four signals into a ranked candidate list. `difficulty` supplies signals 1 (nonlinear) + and 3 (its hotspots whose contract != `cut` are resolved-expensive externals); `hash_signals` supply + signal 2; `surviving` (from `surviving_hostile`) supplies signal 4 — raw prover-hostile primitives + (bitwise / sort / exp / mulDiv) that survive optimization, keyed by function, carrying a category, the + entry methods that reach it, the prover's `summarizable` flag, and a candidate summary. `cut` is the + verified contract (its own methods are NOT "external"). Dependency-tree functions (lib/) are dropped + from the hashing signal unless `include_dependencies` (surviving primitives are kept regardless — they + are in the real problem). `cone_weight` re-weights the build-phase hashing signal by how much code + consumes the result. CVL/ghost hotspots (already-applied summaries) are excluded — they are the + summary, not a summarization target.""" + hotspots = [h for h in difficulty.hotspots if not _is_cvl_ghost(h.function)] + cand: dict[str, Candidate] = {} + + def _bump(key: str, sig: str, score: float, evidence: str): + c = cand.get(key) + if c is None: + cand[key] = Candidate(function=key, signals=(sig,), score=score, evidence=evidence) + else: + if sig not in c.signals: + c.signals = (*c.signals, sig) + c.score += score + c.evidence += " | " + evidence + + # signal 1 (nonlinear) + 3 (external): the difficulty hotspots. The prover's procIds carry a visibility + # marker; three kinds are handled distinctly: + # - unmarked `.m` -> the CUT's OWN external method = a rule subject, never summarized -> DROP + # - unmarked `.m` -> a cross-contract callee (e.g. HubInstanceHarness.previewRemoveByShares) + # -> keep + flag as a resolved external (signal 3) + # - `(internal) .m` -> an inlined internal fn (the ray/bit math). The prover attributes it to the + # CALLING contract, so the same fn recurs under several contexts and (as + # `.fls`) collides with a surviving primitive -> dedup by bare name, + # keeping the highest-contribution instance (hotspots are pct-sorted). + surviving_bare = {raw.split("(", 1)[0].rpartition(".")[2] for raw in (surviving or {})} + seen_internal: set[str] = set() + for h in hotspots: # already excludes CVL/Ghost + if h.pct < NONLINEAR_MIN_PCT: # drop the long, low-contribution tail + continue + internal = h.function.strip().startswith("(internal)") + fn = _strip_procid(h.function) + contract = _contract_of(fn) + bare = fn.rpartition(".")[2] + if contract == cut and not internal: # the CUT's own external method (rule subject) + continue + if bare in surviving_bare: # already a correctly-named surviving candidate + continue + if internal: + if bare in seen_internal: # caller-attribution dup -> keep the top one + continue + seen_internal.add(bare) + _bump(fn, "nonlinear", float(h.pct), + f"{h.pct}% of nonlinear ops" + (f" @{h.location}" if h.location else "")) + if h.location: # the difficulty report carries the location + cand[fn].file, cand[fn].line = _parse_location(h.location) + if contract and contract != cut and not internal: # a genuine cross-contract external call + _bump(fn, "external", 10.0, f"resolved external in {contract} (not the CUT)") + + # signal 2 (hashing/encoding): the AST scan. Dynamic-input hashing (unbounded bytes/string/array) is + # the costly kind; a fixed-size digest (typical EIP-712) is bounded — score it far lower so the noise + # of signature digests sinks below the real candidates rather than being dropped outright. + for h in hash_signals: + if h.is_dependency and not include_dependencies: + continue + cls = "dynamic-input" if h.dynamic_input else "fixed-size" + _bump(h.function, "hashing", 20.0 if h.dynamic_input else 4.0, + f"{'/'.join(h.patterns)} [{cls}] ({h.mutability or 'n/a'} {h.visibility})") + + # signal 4 (surviving hostile primitives): raw prover-hostile primitives still in the sanity run's + # postOptimize TAC — a direct candidate. Score by reach (how many entry methods keep it); attach the + # catalog category, candidate summary, reaching methods, and the prover's summarizable flag. A surviving + # name carries a signature (`C.f(sig)`) while the difficulty/hashing keys are sig-less (`C.f`) — strip it + # so the same function unifies into one candidate. + for raw_fn, rec in (surviving or {}).items(): + fn = raw_fn.split("(", 1)[0] + n = len(rec["reaching_methods"]) + _bump(fn, rec["category"], SURVIVING_SCORE, # the hard-op class IS the signal + f"{rec['category']}, reaches {n} method(s)") + c = cand[fn] + c.reaching_count = n + c.summarizable = rec["summarizable"] + c.candidate_summary = rec["candidate_summary"] + + # cone-of-influence re-weighting for the hashing signal: build-phase cost has no per-function measure, + # so scale a hashing candidate by how much reachable code consumes its result (normalized within the + # candidate set, factor in [1, 2], so it stays comparable to the measured nonlinear pct). A hash whose + # id threads the protocol rises; a leaf digest (tiny cone) stays low. + if cone_weight: + hashing = [c for c in cand.values() if "hashing" in c.signals] + mx = max((cone_weight.get(c.function, 0) for c in hashing), default=0) or 1 + for c in hashing: + w = cone_weight.get(c.function, 0) + c.score *= 1 + w / mx + c.evidence += f" | cone={w}" + + ranked = sorted(cand.values(), key=lambda c: (-c.score, c.function)) # score desc, name for stable ties + kept: list[Candidate] = [] + per_cat: dict[str, int] = {} + for c in ranked: # score-ordered, so per category we keep the top ones + cat = _bucket(c) + if per_cat.get(cat, 0) < MAX_PER_CATEGORY.get(cat, 0): + per_cat[cat] = per_cat.get(cat, 0) + 1 + kept.append(c) + return DetectionReport(candidates=kept, dropped=len(ranked) - len(kept)) + + +def _survives(function: str, surviving_set: set[str], survivors_bare: set[str]) -> bool: + """Whether `function` (a scan_ast candidate) reaches SMT per the prover's surviving set. A host-less + free (file-level) function has no contract host in the AST (e.g. `computeBaseHash`), but the prover + attributes it to an arbitrary host in the surviving set (e.g. `Ownable.computeBaseHash`) — so it's + matched by bare name. A hosted candidate matches on its exact qualified name, so a genuinely- + unreachable `CTHelpers.getConditionId` is NOT resurrected by a same-named method on another contract + (`survivors_bare` is `{name after last '.'}` over the surviving set).""" + return function in surviving_set or ("." not in function and function in survivors_bare) + + +def _surviving_reach(graphs: list[dict]) -> set[str]: + """The reachability set (sig-stripped `Contract.fn`) over the postOptimize surviving graphs — the + functions that actually reach SMT. This is the authoritative signal-2 gate.""" + return {name.split("(", 1)[0] for g in graphs for name, _ in _surviving_names(g)} + + +def detect(job_url: str | None = None, *, ast_path: str | Path | None = None, + conf: str | Path | None = None, cut: str, solc_dir: str | Path | None = None, + include_dependencies: bool = False, + external_call_graph: str | Path | None = None, + surviving_graphs: list[dict] | None = None, + sources_root: str | Path | None = None) -> DetectionReport: + """Orchestrate the detector. `job_url` (optional) supplies the difficulty report (signals 1+3); with + none, only the static signal-2 (hashing) runs. The AST is resolved by `ensure_ast` (`ast_path` if + given, else generated from `conf`). `cut` is the verified contract name. + + `surviving_graphs` are the prover's postOptimize SurvivingCallGraph dumps: they drive signal 4 (the + surviving hostile primitives) AND give the authoritative signal-2 reachability gate (the functions that + actually reach SMT). Absent them, signal-2 falls back to AST + `external_call_graph` reachability from + the CUT (see `reachable_from_main`). `sources_root` (the project root the AST paths are relative to, + defaulting to the conf's directory) lets each candidate's start line be resolved from the source.""" + ast = ensure_ast(ast_path, conf=conf, solc_dir=solc_dir) + if sources_root is None and conf is not None: + sources_root = Path(conf).parent + hash_signals = scan_ast(ast) + surviving = surviving_hostile(surviving_graphs or []) + surviving_set = _surviving_reach(surviving_graphs) if surviving_graphs else None + cone: dict[str, int] = {} + reachable: set[str] = set() + edges: dict[str, set[str]] = {} + sigs: dict[str, tuple[str, bool, bool, bool]] = {} + merged = _unit_declaring(ast, cut) # the CUT's compilation unit, built once + if merged is not None: + edges, roots, by_name, sizes = _ast_call_graph(merged, cut) + if external_call_graph is not None: + _add_external_edges(edges, by_name, external_call_graph) + reachable = _bfs(edges, roots) # for cone (rank) + the ecg fallback gate + cone = _cone_weights(edges, reachable, sizes) + sigs = _signatures(merged) # for caller-boundary expressibility + if surviving_set: # authoritative: exactly what reached SMT + survivors_bare = {n.rpartition(".")[2] for n in surviving_set} + hash_signals = [h for h in hash_signals if _survives(h.function, surviving_set, survivors_bare)] + elif merged is not None and external_call_graph is not None: # fallback: complete cross-contract edges + hash_signals = [h for h in hash_signals if h.function in reachable] + difficulty = fetch_difficulty(job_url, limit=None) if job_url else DifficultyReport() # detector filters + report = detect_from(hash_signals, difficulty, cut=cut, cone_weight=cone, + include_dependencies=include_dependencies, surviving=surviving) + # A hashing or surviving candidate is the cost LEAF (the exact function to summarize) — offer caller + # boundaries too: where else the agent can place the summary (a clean-signature caller subsumes the + # leaf and is often more feasible to model — see Boundary). + bounds_reach = surviving_set if surviving_set else (reachable or None) + if edges: + # nonlinear-only candidates: descend to the shared nonlinear PRIMITIVE they inline, and count how + # many candidates share each (fan-in) — a primitive shared across many methods is the real target. + # Filter by AST reachability, NOT the surviving set: library `using`-for primitives (e.g. + # MathLib.mulDivDown) get no INTERNAL_FUNC_START annotation, so they are absent from the surviving + # set even though they are genuinely inlined into a surviving method. + nl = [c.function for c in report.candidates + if "nonlinear" in c.signals and "hashing" not in c.signals + and not any(s in _PRIMITIVE_CATEGORIES for s in c.signals)] + prim_reach = {m: _descend_to_prims(edges, m, reachable or None) for m in nl} + fanin: dict[str, int] = {} + for prims in prim_reach.values(): + for p in prims: + fanin[p] = fanin.get(p, 0) + 1 + for c in report.candidates: + if "hashing" in c.signals or any(s in _PRIMITIVE_CATEGORIES for s in c.signals): # walk UP to a caller + c.boundaries = _caller_boundaries(edges, sigs, c.function, bounds_reach) + elif "nonlinear" in c.signals: # descend DOWN to the shared nonlinear primitive + targets = sorted(prim_reach.get(c.function, {}).items(), + key=lambda kv: (-fanin.get(kv[0], 0), kv[1], kv[0]))[:4] + c.boundaries = [] + for p, d in targets: + sig, expressible, mutating, _ext = sigs.get(p, (p, False, False, False)) + if not expressible: # a shared primitive we can't express as a value is no target + continue + c.boundaries.append(Boundary(p, d, sig, mutating, direction="down", shared=fanin.get(p, 1))) + # attach each candidate's source location (file always; line when the source is readable) + locations = _function_locations(ast, sources_root) + for c in report.candidates: + if c.function in locations: + c.file, c.line = locations[c.function] + # attach a signature + view/pure flag (the agent needs the param/return types to write the summary, and + # `mutating` to know a value-summary would erase side effects) — the same AST map the boundaries use. + # Exact qualified match, else bare name: the difficulty report attributes an inlined fn to its CALLING + # contract (`HubInstanceHarness.calculatePremiumRay`), which resolves by the bare `calculatePremiumRay`. + bare_sigs = {q.rpartition(".")[2]: v for q, v in sigs.items()} + for c in report.candidates: + s = sigs.get(c.function) or bare_sigs.get(c.function.rpartition(".")[2]) + if s: + c.signature, _expr, c.mutating, _ext = s + return report + + +# ---------------------------------------------------------------- reachability-from-main (signal-2 gate) +# Signal 2 sweeps EVERY function in the scene; most are noise (an EIP-712 digest in a module the CUT never +# reaches). We prune to functions reachable from the CUT's external/public entry points over the real call +# graph. Edges come from two authoritative sources, never guessed: DIRECT internal/library calls from the +# solc AST (`referencedDeclaration`), and CROSS-CONTRACT calls from the prover's `externalCallGraph.json` +# (linking + dispatch — a static AST can't resolve which impl an interface call hits). Reachability +# OVER-approximates (dispatch matched by selector across the scene), which is the safe direction for a +# prune: we keep a maybe-reachable candidate rather than drop a real one. +def _span3(node: dict) -> tuple[int, int, int] | None: + """`src` = "offset:length:fileId" -> (start, end, fileId). fileId distinguishes the source files that + share one compilation unit's node-id space.""" + src = node.get("src") + if not isinstance(src, str): + return None + try: + off, length, fid = src.split(":") + return int(off), int(off) + int(length), int(fid) + except ValueError: + return None + + +def _enclosing(off: int, spans: list) -> str | None: + """The tightest (start, end, name) span containing `off` — the enclosing contract/function name.""" + best = None + for s, e, name in spans: + if s <= off < e and (best is None or (e - s) < (best[1] - best[0])): + best = (s, e, name) + return best[2] if best else None + + +def _unit_declaring(ast_path: str | Path, cut: str) -> dict | None: + """Merge (by node id) all nodes of the compilation unit that DECLARES contract `cut`. Node ids are + unique within a unit's single solc id space, so `referencedDeclaration` resolves inside the merge; the + CUT's unit already contains every file it imports (its full reachable closure).""" + for _rel, pdata in stream_raw_units(Path(ast_path)): + if not isinstance(pdata, dict): + continue + merged: dict[str, dict] = {} + declares = False + for _abs, nodes in pdata.items(): + if not isinstance(nodes, dict): + continue + for nid, node in nodes.items(): + if isinstance(node, dict): + merged[nid] = node + if node.get("nodeType") == "ContractDefinition" and node.get("name") == cut: + declares = True + if declares: + return merged + return None + + +def _ast_call_graph(merged: dict, cut: str): + """From the CUT's merged compilation unit build: internal call edges (caller qual -> callee qual via + FunctionCall `referencedDeclaration`), the CUT's external/public entry points (BFS roots), and a + name->quals index (to resolve dispatch selectors to scene methods). `qual` = "Contract.fn".""" + contracts_by_fid: dict[int, list] = {} + for node in merged.values(): + if node.get("nodeType") == "ContractDefinition": + sp = _span3(node) + if sp: + contracts_by_fid.setdefault(sp[2], []).append((sp[0], sp[1], node.get("name") or "")) + + fndefs_by_fid: dict[int, list] = {} + qual_by_id: dict[str, str] = {} + roots: set[str] = set() + by_name: dict[str, set[str]] = {} + sizes: dict[str, int] = {} # qual -> body size (source bytes), for cone weight + for nid, node in merged.items(): + if node.get("nodeType") != "FunctionDefinition": + continue + sp = _span3(node) + if sp is None: + continue + cname = _enclosing(sp[0], contracts_by_fid.get(sp[2], [])) or "" + fname = node.get("name") or "" + qual = f"{cname}.{fname}" if cname else fname + fndefs_by_fid.setdefault(sp[2], []).append((sp[0], sp[1], qual)) + qual_by_id[nid] = qual + sizes[qual] = max(sizes.get(qual, 0), sp[1] - sp[0]) + if fname: + by_name.setdefault(fname, set()).add(qual) + if cname == cut and node.get("visibility") in ("external", "public"): + roots.add(qual) + + edges: dict[str, set[str]] = {} + for node in merged.values(): + if node.get("nodeType") != "FunctionCall": + continue + ref = (node.get("expression") or {}).get("referencedDeclaration") + callee = qual_by_id.get(str(ref)) if ref is not None else None + if callee is None: + continue + sp = _span3(node) + if sp is None: + continue + caller = _enclosing(sp[0], fndefs_by_fid.get(sp[2], [])) + if caller: + edges.setdefault(caller, set()).add(callee) + return edges, roots, by_name, sizes + + +def _add_external_edges(edges: dict, by_name: dict, external_call_graph: str | Path) -> None: + """Fold the prover's resolved/dispatch cross-contract edges into `edges`. For a RESOLVED target we add + `caller -> targetContract.selectorMethod`; for a dispatch/symbolic target (contract unknown) we add + `caller -> X.selectorMethod` for every scene contract X declaring that method (selector-matched + over-approx). Selectors give the callee method name; unresolved targets with no scene match are skipped + (they leave the scene — the linker's concern, not ours).""" + data = json.loads(Path(external_call_graph).read_text()) + known = {q for qs in by_name.values() for q in qs} + for host, call_edges in data.items(): + for e in call_edges: + caller = f"{host}.{e['caller'].split('(', 1)[0]}" + sel_names = [s["signature"].split("(", 1)[0] for s in e.get("selectors", []) + if s.get("signature")] + for t in e.get("targets", []): + contract = t.get("contract") + for sel in sel_names: + if contract: + callee = f"{contract}.{sel}" + if callee in known: + edges.setdefault(caller, set()).add(callee) + else: # dispatch: any scene method of that name + for callee in by_name.get(sel, ()): + edges.setdefault(caller, set()).add(callee) + + +def _build_graph(ast_path: str | Path, cut: str, external_call_graph: str | Path | None): + """(edges, roots, sizes) for the CUT's compilation unit — AST internal edges (+ prover external/dispatch + edges when given). None if the CUT's unit isn't found. Shared by reachability and the cone weight.""" + merged = _unit_declaring(ast_path, cut) + if merged is None: + return None + edges, roots, by_name, sizes = _ast_call_graph(merged, cut) + if external_call_graph is not None: + _add_external_edges(edges, by_name, external_call_graph) + return edges, roots, sizes + + +def _bfs(edges: dict, roots) -> set[str]: + seen = set(roots) + stack = list(roots) + while stack: + cur = stack.pop() + for nxt in edges.get(cur, ()): + if nxt not in seen: + seen.add(nxt) + stack.append(nxt) + return seen + + +def _cone_weights(edges: dict, reachable: set, sizes: dict) -> dict[str, int]: + """Per reachable function, the total body size of its TRANSITIVE CONSUMERS — the code that (transitively) + calls it, hence reasons about its output. Invert the edges (callee -> callers) and walk from f toward its + callers, staying inside `reachable`.""" + consumers: dict[str, set[str]] = {} + for caller, callees in edges.items(): + for callee in callees: + consumers.setdefault(callee, set()).add(caller) + out: dict[str, int] = {} + for f in reachable: + seen: set[str] = set() + stack = [c for c in consumers.get(f, ()) if c in reachable] + while stack: + c = stack.pop() + if c in seen: + continue + seen.add(c) + stack.extend(x for x in consumers.get(c, ()) if x in reachable and x not in seen) + out[f] = sum(sizes.get(g, 0) for g in seen) + return out + + +_NON_EXPRESSIBLE_ELEM = {"bytes", "string"} # dynamic byte blobs; fixed `bytesN` / value types are fine + + +def _param_infos(node: dict, key: str) -> list[tuple[str, object]]: + """(typeString for display, typeName node for the structural check) per `parameters`/`returnParameters`.""" + ps = ((node.get(key) or {}).get("parameters")) or [] + return [(((p.get("typeDescriptions") or {}).get("typeString") or ""), p.get("typeName")) for p in ps] + + +def _expressible_typename(tn: object, merged: dict, seen: frozenset = frozenset()) -> bool: + """Whether a solc `typeName` is modelable as a typed CVL summary value — walked RECURSIVELY, so a blob + nested inside an array or struct is caught (unlike a flat typeString check). Expressible: value types, + UDVTs, enums, contracts; arrays (fixed or dynamic) of an expressible element; structs whose fields are + all expressible. NOT expressible, wherever nested: dynamic `bytes`/`string` (abi-encoded blobs — the + point of an opaque boundary), mappings, and function types.""" + if not isinstance(tn, dict): + return False + nt = tn.get("nodeType") + if nt == "ElementaryTypeName": + return (tn.get("name") or "") not in _NON_EXPRESSIBLE_ELEM + if nt == "ArrayTypeName": + return _expressible_typename(tn.get("baseType"), merged, seen) + if nt == "UserDefinedTypeName": + ref = tn.get("referencedDeclaration") + decl = merged.get(str(ref)) if ref is not None else None + if not isinstance(decl, dict): + return True # external/interface type not in unit -> value-like + if decl.get("nodeType") == "StructDefinition": + sid = str(ref) + if sid in seen: # recursive struct: no new blob on this cycle + return True + return all(_expressible_typename((m or {}).get("typeName"), merged, seen | {sid}) + for m in (decl.get("members") or [])) + return True # enum / UDVT / contract -> value-like + if nt in ("Mapping", "FunctionTypeName"): + return False + return False # unknown node -> conservative + + +def _signatures(merged: dict) -> dict[str, tuple[str, bool, bool, bool]]: + """qual -> (readable `fn(pt,..) -> rt,..` signature, is-CVL-expressible, is-mutating, is-external). + Expressible = has a return AND every param/return type is CVL-expressible (see `_expressible_typename`); + a void function has no value to model, so it is not expressible. Mutating = stateMutability not + view/pure (summarizing it as a value erases side effects). External = public/external visibility — an + entry point (the rule's subject), never an internal summarization boundary.""" + contracts_by_fid: dict[int, list] = {} + for node in merged.values(): + if node.get("nodeType") == "ContractDefinition": + sp = _span3(node) + if sp: + contracts_by_fid.setdefault(sp[2], []).append((sp[0], sp[1], node.get("name") or "")) + out: dict[str, tuple[str, bool, bool, bool]] = {} + for node in merged.values(): + if node.get("nodeType") != "FunctionDefinition": + continue + sp = _span3(node) + if sp is None: + continue + cname = _enclosing(sp[0], contracts_by_fid.get(sp[2], [])) or "" + fname = node.get("name") or "" + qual = f"{cname}.{fname}" if cname else fname + params = _param_infos(node, "parameters") + rets = _param_infos(node, "returnParameters") + sig = f"{fname}({', '.join(t for t, _ in params)})" + ( + f" -> {', '.join(t for t, _ in rets)}" if rets else "") + has_return = bool(rets) + types_ok = all(_expressible_typename(tn, merged) for _, tn in params + rets) + mutating = (node.get("stateMutability") or "") not in ("view", "pure") + is_external = (node.get("visibility") or "") in ("external", "public") + out[qual] = (sig, has_return and types_ok, mutating, is_external) + return out + + +def _caller_boundaries(edges: dict, sigs: dict, leaf: str, reachable: set[str] | None, + max_hops: int = 4, limit: int = 4) -> list[Boundary]: + """Walk UP the call graph from `leaf` (a detected hasher) and return caller boundaries — places to + summarize instead of the leaf — ranked expressible-first then nearest. Restricted to `reachable` + (host-less free functions matched by bare name, as in the gate) when given, so suggestions are real.""" + consumers: dict[str, set[str]] = {} + for caller, callees in edges.items(): + for callee in callees: + consumers.setdefault(callee, set()).add(caller) + bare = {n.rpartition(".")[2] for n in reachable} if reachable is not None else set() + hops: dict[str, int] = {} + frontier, depth = {leaf}, 0 + while frontier and depth < max_hops: + depth += 1 + nxt: set[str] = set() + for f in frontier: + for c in consumers.get(f, ()): + if c not in hops and c != leaf: + hops[c] = depth + nxt.add(c) + frontier = nxt + out: list[Boundary] = [] + for q, h in hops.items(): + if reachable is not None and not _survives(q, reachable, bare): + continue + sig, expressible, mutating, is_external = sigs.get(q, (q, False, False, False)) + if not expressible or is_external: # keep only internal, value-expressible (actionable) boundaries + continue + out.append(Boundary(q, h, sig, mutating)) + # view/pure over state-changing, then nearest — the cleanest, safest boundary first + out.sort(key=lambda b: (b.mutating, b.hops, b.function)) + return out[:limit] + + +# Nonlinear PRIMITIVES: arg-based math-library fns (Morpho MathLib, Solmate FixedPointMathLib, OZ Math, +# ds-math, ...). A nonlinear hotspot METHOD usually just INLINES one of these; the difficulty report +# attributes the ops to the method, so descending to the shared primitive names the real, arg-based +# over-approx target. Name-based for now — extensible to an AST nonlinear-op body scan. +_NONLINEAR_PRIMS = { + "mulDiv", "mulDivDown", "mulDivUp", "fullMulDiv", "fullMulDivUp", "mulDivRoundingUp", + "mulWad", "mulWadDown", "mulWadUp", "divWad", "divWadDown", "divWadUp", "sMulWad", "sDivWad", + "rpow", "wpow", "pow", "rmul", "rdiv", "wmul", "wdiv", "sqrt", "cbrt", "exp", "expWad", "ln", "lnWad", +} + + +def _is_nonlinear_prim(qual: str) -> bool: + return qual.rpartition(".")[2] in _NONLINEAR_PRIMS + + +def _descend_to_prims(edges: dict, method: str, reachable: set[str] | None, + max_depth: int = 4) -> dict[str, int]: + """BFS DOWN the call graph from `method`; return {nonlinear-primitive qual -> min call depth} for the + primitives it (transitively) inlines. Restricted to `reachable` (free functions bare-name matched, as + in the gate) so dead code is never suggested.""" + bare = {n.rpartition(".")[2] for n in reachable} if reachable is not None else set() + prims: dict[str, int] = {} + seen = {method} + frontier, depth = {method}, 0 + while frontier and depth < max_depth: + depth += 1 + nxt: set[str] = set() + for f in frontier: + for callee in edges.get(f, ()): + if callee in seen: + continue + seen.add(callee) + nxt.add(callee) + if (_is_nonlinear_prim(callee) and callee not in prims + and (reachable is None or _survives(callee, reachable, bare))): + prims[callee] = depth + frontier = nxt + return prims + + +def reachable_from_main(ast_path: str | Path, cut: str, external_call_graph: str | Path | None = None) -> set[str]: + """The set of `Contract.fn` reachable from the CUT's external/public entry points, over the combined + call graph (AST internal edges + prover external/dispatch edges). Empty if the CUT's compilation unit + can't be found (caller treats empty as "don't prune").""" + g = _build_graph(ast_path, cut, external_call_graph) + return _bfs(g[0], g[1]) if g else set() + + +def cone_weights(ast_path: str | Path, cut: str, external_call_graph: str | Path | None = None) -> dict[str, int]: + """Heuristic cone-of-influence weight per reachable function — the size of the code that consumes its + output (`_cone_weights`). The prover exposes no COI, so we approximate it over the call graph; it + over-approximates the true data-flow cone but orders candidates by how widely their result propagates + (a hash whose id threads the protocol outranks a leaf digest).""" + g = _build_graph(ast_path, cut, external_call_graph) + if g is None: + return {} + edges, roots, sizes = g + return _cone_weights(edges, _bfs(edges, roots), sizes) diff --git a/summarization_detector/difficulty.py b/summarization_detector/difficulty.py new file mode 100644 index 00000000..5fe81f8d --- /dev/null +++ b/summarization_detector/difficulty.py @@ -0,0 +1,143 @@ +"""Fetch a completed job's difficulty signal WITHOUT the whole zipOutput or statsdata.json. + +Reads the prover's own ranked NONLINEARITY HOTSPOTS from the run (via POU — treeView only, no tarball): + + rule_live_statistics_*.json -> "nonlinearity hotspots" node -> functions ranked by % contribution to + nonlinear ops, each carrying a source file:line. + +A function appears here ONLY IF its body was INLINED into the SMT problem — a summarized or havoc'd call +contributes no nonlinear ops. So the hotspot list is the real, in-problem, expensive bytecode: the +summarization candidates. + +Note on call resolutions: `get_call_resolutions` is NOT used to find inlined calls. That table is built +only from applied-summary annotations (EVMVerifier CallResolutionTable.kt:161 -> +TACProgram.topoSortedSummaryStart), so an inlined call has NO row; POU additionally filters it to +unresolved (`[?]`) callees (prover_output_utility tree_parser.py:136). It reports the OPPOSITE of inlined — +already-havoc'd unresolved externals — so it cannot surface a summarization candidate. + +`statsdata.json` is avoided (can be huge, and it carries the same nonlinear-op scores WITHOUT source +locations — those are joined from the call graph only in the rendered treeView). Best-effort: any failure +(missing POU / auth / network / schema drift) returns an empty report. +""" + +import json +import re +from dataclasses import dataclass, field + +_MAX_HOTSPOTS = 8 # a ranked pointer, not a dump +# Schema mirrors EVMVerifier's producers (Kotlin, cited so drift is traceable): +# report/LiveCheckInfoNode.kt -> node fields {label, value, children, jumpToDefinition, +# severity, hSev}; jumpToDefinition = TreeViewLocation +# {file, start:{line,col}, end:{line,col}}. +# statistics/data/SingleDifficultyStats.kt:219-259 -> the "nonlinearity hotspots" node: +# parent label = "nonlinearity hotspots" +# child label = "function: $procId" +# value = "contrib. to nonlinear ops: $x %" [ \n "contrib. to max polyn. degree: $y %"] +# jumpToDefinition = callGraphInfo.procIdToSourceLocation[procId] +# (the prover already keeps only procs with nlOps>10 || polydeg>5, sorted by nlOps+polydeg). +_HOTSPOTS_NODE_LABEL = "nonlinearity hotspots" +_HOTSPOT_FN_RE = re.compile(r"function:\s*(?P.+)") # $procId (may contain spaces/quotes) +_HOTSPOT_PCT_RE = re.compile(r"nonlinear ops:\s*(?P\d+)\s*%") + + +@dataclass +class Hotspot: + function: str # e.g. "SomeLib.someNonlinearFn" (procId) + pct: int # % contribution to the rule's nonlinear ops + location: str # "file:line" or "" + + +@dataclass +class DifficultyReport: + hotspots: list[Hotspot] = field(default_factory=list) + + def is_empty(self) -> bool: + return not self.hotspots + + def format(self) -> str: + """Render the ranked hotspots as a compact, source-located block plus a summarization playbook. + Every listed function has its real body INLINED in the SMT problem (that is why it contributes + nonlinear ops).""" + if self.is_empty(): + return "" + out = [" nonlinearity hotspots (prover difficulty report — % of the rule's nonlinear ops; each " + "function's real body is INLINED in the problem):"] + for h in self.hotspots: + at = f" @{h.location}" if h.location else "" + out.append(f" {h.pct:3d}% {h.function}{at}") + out.append( + " -> NONDET the OFF-PATH hotspots (result not read by the checked output) to delete their " + "nonlinear subproblem; if a hotspot stays inlined despite a `_.fn` wildcard NONDET, it is a " + "LINKED target — summarize the CONCRETE contract: `function .(...) external " + "returns (...) => NONDET;` (add_nondet with contract=). For ON-PATH math (the CUT " + "method itself, or a getter the glue observes), mirror the scene's existing summary form so " + "equality is congruence-trivial — do NOT NONDET it.") + return "\n".join(out) + + +def _parse_hotspots(node, out: dict[str, Hotspot]) -> None: + """Walk a rule_live_statistics tree; collect the children of every 'nonlinearity hotspots' node.""" + if isinstance(node, dict): + if node.get("label") == _HOTSPOTS_NODE_LABEL: + for c in node.get("children", []) or []: + mfn = _HOTSPOT_FN_RE.search(str(c.get("label", ""))) + mpct = _HOTSPOT_PCT_RE.search(str(c.get("value", ""))) + if not (mfn and mpct): + continue + fn, pct = mfn.group("fn").strip(), int(mpct.group("pct")) + jd = c.get("jumpToDefinition") or {} + loc = "" + if isinstance(jd, dict) and jd.get("file"): + loc = f"{jd['file']}:{jd.get('start', {}).get('line')}" + prev = out.get(fn) + if prev is None or pct > prev.pct: # dedupe across split rules, keep the worst + out[fn] = Hotspot(function=fn, pct=pct, location=loc) + for c in node.get("children", []) or []: + _parse_hotspots(c, out) + elif isinstance(node, list): + for c in node: + _parse_hotspots(c, out) + + +_LIVE_STATS_RE = re.compile(r"rule_live_statistics_(\d+)\.json") +_PROBE_MAX = 64 # fallback if the status doesn't reference the live-stats files by name + + +def _live_stats_indices(api, job_url: str) -> list[int]: + """The rule_live_statistics_*.json indices for this job. POU's `fetch_job_treeview` bulk-download + does NOT include these files (only rule_output_* + treeViewStatus), but they ARE served per-file by + `fetch_treeview_output_by_filename`. Enumerate them from the treeViewStatus (which references the + per-rule output files); fall back to a bounded probe if none are named.""" + try: + idx = {int(n) for n in _LIVE_STATS_RE.findall(json.dumps(api.get_treeview_status(job_url)))} + if idx: + return sorted(idx) + except Exception: + pass + return list(range(_PROBE_MAX)) + + +def fetch_difficulty(job_url: str, limit: int | None = _MAX_HOTSPOTS) -> DifficultyReport: + """Best-effort difficulty report for a completed job: the prover's ranked nonlinearity hotspots + (function, % of nonlinear ops, source file:line). Returns an empty report on any error. Uses ONLY + the treeView `rule_live_statistics_*.json` files (fetched per-file via POU's public API) — never + statsdata.json or the output tarball. `limit` caps the returned hotspots (default `_MAX_HOTSPOTS`); + pass ``None`` for the full ranked set.""" + report = DifficultyReport() + if not job_url: + return report + try: + from prover_output_utility import ProverOutputAPI + api = ProverOutputAPI(use_local=False) + except Exception: + return report + + hs: dict[str, Hotspot] = {} + for n in _live_stats_indices(api, job_url): + try: + _parse_hotspots(api.fetch_treeview_output_by_filename(job_url, f"rule_live_statistics_{n}.json"), hs) + except Exception: + continue # index not present (expected when probing) / transient fetch error + ranked = sorted(hs.values(), key=lambda h: h.pct, reverse=True) + report.hotspots = ranked if limit is None else ranked[:limit] + return report diff --git a/summarization_detector/difficulty_profile.py b/summarization_detector/difficulty_profile.py new file mode 100644 index 00000000..d5e3def1 --- /dev/null +++ b/summarization_detector/difficulty_profile.py @@ -0,0 +1,270 @@ +"""Difficulty PROFILER — from a completed prover run, find the slow rules and attribute WHERE the prover +time goes, at source granularity. + +The POST-HOC counterpart to `detect.py` (the static, before-the-rules predictor): given a real-property +run, it reads the prover's difficulty tree per slow rule (`rule_live_statistics_*.json`: nonlinearity / +path-count / memory hotspots, each with a source `file:line`), rolls the hotspots up by function, and +classifies each function by kind: + + cut a function of the contract under test + library an inlined library (not a scene contract) + external a linked/real dependency contract + cvl-model an already-applied CVL summary (ghost) — contributes ~0 + +The contract under test and the scene's linked contracts are read from the job's treeViewStatus, so no +protocol-specific configuration is baked in. Reuses POU (`ProverOutputAPI`) as the `difficulty` module +does; best-effort and tolerant of schema drift. +""" + +import re +from dataclasses import asdict, dataclass, field + +# `duration` in treeViewStatus nodes is WALL SECONDS. A rule is "slow" (and gets profiled) when its +# duration reaches this threshold OR its status is TIMEOUT — the TIMEOUT check catches a rule that hit the +# run's global timeout whatever that cap was. Overridable via --min-minutes. +_DEFAULT_MIN_SECONDS = 300 # 5 min +_MAX_HOTSPOTS_PER_RULE = 4 # a ranked pointer per rule, not a dump +_HOTSPOT_PARENTS = { # difficulty-tree nodes whose children are per-function hotspots + "nonlinearity hotspots": "nl", + "path count hotspots": "path", + "memory complexity hotspots": "mem", +} +_FN_RE = re.compile(r"function:\s*(?P.+)", re.S) +_PCT_RE = re.compile(r"(\d+)\s*%") +# procId of an applied CVL summary, e.g. "CVL/Ghost Function 'cvlPrice(id)'". +_CVL_PREFIXES = ("CVL/", "CVL ", "cvl") + + +@dataclass +class Hotspot: + kind: str # "nl" | "path" | "mem" + function: str # procId, e.g. "Vault.computeAccountData" + contract: str # the part before the first '.', or "" for a CVL ghost + pct: int # % contribution to this rule's ops of that kind + location: str # "file:line" or "" + klass: str # "cut" | "library" | "external" | "cvl-model" | "unknown" + + +@dataclass +class SlowRule: + name: str + status: str + minutes: float + split_progress: object # int % or None + path_count: str # e.g. "approx. 2^127" + nonlinearity: str # e.g. "nonlinear ops: 286 max polyn. degree: 6" + hotspots: list[Hotspot] = field(default_factory=list) + + +@dataclass +class FunctionRollup: + function: str + klass: str + location: str + nl_pct_sum: int # summed nonlinear-op % across the slow rules it dominates (a rank key) + rules: int # how many slow rules it is a top hotspot in + + +@dataclass +class ProfileReport: + job: str + cut: str + spec: str + n_rules: int + slow: list[SlowRule] = field(default_factory=list) + by_class: dict = field(default_factory=dict) # klass -> summed nl% + by_function: list[FunctionRollup] = field(default_factory=list) + + def to_dict(self) -> dict: + return asdict(self) + + def format(self) -> str: + out = [f"difficulty profile — {self.cut} ({self.job})", + f" spec: {self.spec} rules: {self.n_rules} slow (>=threshold or TIMEOUT): {len(self.slow)}"] + for r in self.slow: + out.append(f" [{r.status:8}] {r.minutes:6.1f}min split={r.split_progress}% {r.name[-64:]}") + out.append(f" path={r.path_count} nl={r.nonlinearity[:46]}") + for h in r.hotspots: + out.append(f" {h.kind:4} {h.pct:3d}% [{h.klass:9}] {h.function[:52]} {h.location}") + out.append(" --- where the nonlinear ops go (by class) ---") + for k, v in sorted(self.by_class.items(), key=lambda x: -x[1]): + out.append(f" {v:5d}% {k}") + out.append(" --- top functions by prover cost (ranked) ---") + for f in self.by_function[:12]: + out.append(f" {f.nl_pct_sum:5d}% [{f.klass:9}] {f.function} ({f.rules} rules) {f.location}") + return "\n".join(out) + + +def _loc(node: dict) -> str: + jd = node.get("jumpToDefinition") + if isinstance(jd, dict): + return f"{jd.get('file')}:{jd.get('start', {}).get('line')}" + if isinstance(jd, list) and jd: + return f"{jd[0].get('file')}:{jd[0].get('start', {}).get('line')}" + return "" + + +def _classify(function: str, cut: str, scene_contracts: set[str]) -> tuple[str, str]: + """Return (contract, klass) — `klass` is the hotspot's kind: cut / library / external / cvl-model.""" + fn = function.strip().strip("'\"") + if fn.startswith(_CVL_PREFIXES): + return "", "cvl-model" # an already-applied summary — nothing to do + contract = fn.split(".", 1)[0].split("(", 1)[0].strip() if "." in fn else "" + if not contract: + return "", "unknown" + if contract == cut: + return contract, "cut" # a function OF the contract under test + if contract in scene_contracts: + return contract, "external" # a linked/real dependency contract in the scene + return contract, "library" # not a scene contract -> an inlined library + + +def _parse_difficulty_tree(tree, cut: str, scene_contracts: set[str]) -> tuple[dict, list[Hotspot]]: + """Walk a rule_live_statistics tree: collect the top metrics and the per-function hotspot children.""" + metrics: dict = {} + hotspots: list[Hotspot] = [] + + def walk(n, parent_hotspot_kind=None): + if not isinstance(n, dict): + return + lbl = (n.get("label") or "").strip() + val = (n.get("value") or "").strip() + low = lbl.lower() + if low.startswith("path count") and val and "path_count" not in metrics: + metrics["path_count"] = val + elif low.startswith("nonlinearity") and val and "nonlinearity" not in metrics: + metrics["nonlinearity"] = val.replace("\n", " ") + elif low.startswith("memory complexity") and val and "memory" not in metrics: + metrics["memory"] = val.replace("\n", " ") + if parent_hotspot_kind: + m = _FN_RE.match(lbl) + if m: + fn = m.group(1).strip() + pctm = _PCT_RE.search(val) + pct = int(pctm.group(1)) if pctm else 0 + contract, klass = _classify(fn, cut, scene_contracts) + hotspots.append(Hotspot(parent_hotspot_kind, fn, contract, pct, _loc(n), klass)) + child_kind = _HOTSPOT_PARENTS.get(low, parent_hotspot_kind if not low.endswith("hotspots") else None) + for c in (n.get("children") or []): + walk(c, child_kind) + + walk(tree if isinstance(tree, dict) else {"children": tree}) + return metrics, hotspots + + +def _iter_slow_leaves(rules, min_seconds): + """Recurse the rule tree; yield leaf nodes that carry a live-stats file and are slow / TIMEOUT. + Parametric children are prefixed with their parent's name; induction scaffolding names are skipped.""" + def rec(n, prefix=""): + raw = n.get("name") or n.get("label") or "?" + name = f"{prefix}::{raw}" if prefix else raw + lci = n.get("LiveCheckInfo") + dur = n.get("duration") or 0 + st = n.get("status") + if isinstance(lci, str) and lci.endswith(".json") and (dur >= min_seconds or st == "TIMEOUT"): + yield {"name": name, "status": st, "duration": dur, "lci": lci, + "split": n.get("splitProgress")} + next_prefix = prefix if str(raw).startswith("Induction") else name + for c in (n.get("children") or []): + yield from rec(c, next_prefix) + for r in rules: + yield from rec(r) + + +def profile_job(job_url: str, *, min_seconds: int = _DEFAULT_MIN_SECONDS, cut: str | None = None, + api=None) -> ProfileReport: + """Profile one prover job: its slow rules and where (by source function/class) the nonlinear ops go. + Best-effort — returns whatever it can fetch. `cut` overrides the contract-under-test (else read from + the job's treeViewStatus). Pass a shared `api` to reuse a connection across jobs.""" + if api is None: + from .sources import _aiss_env_for + _aiss_env_for(job_url) + from prover_output_utility import ProverOutputAPI + api = ProverOutputAPI(use_local=False) + tv = api.get_treeview_status(job_url) + rules = tv.get("rules", []) if isinstance(tv, dict) else [] + cut = str(cut or (tv.get("contract") if isinstance(tv, dict) else "") or "") + spec = tv.get("spec", "") if isinstance(tv, dict) else "" + scene_contracts = {c.get("name") for c in (tv.get("availableContracts") or []) if c.get("name")} + + slow: list[SlowRule] = [] + by_class: dict = {} + fn_agg: dict = {} # function -> [nl_pct_sum, klass, location, rule_count] + for leaf in sorted(_iter_slow_leaves(rules, min_seconds), key=lambda x: -x["duration"]): + try: + tree = api.fetch_treeview_output_by_filename(job_url, leaf["lci"]) + except Exception: + continue + metrics, hs = _parse_difficulty_tree(tree, cut, scene_contracts) + nl = sorted([h for h in hs if h.kind == "nl"], key=lambda h: h.pct, reverse=True) + pc = sorted([h for h in hs if h.kind == "path"], key=lambda h: h.pct, reverse=True)[:2] + top = nl[:_MAX_HOTSPOTS_PER_RULE] + pc + slow.append(SlowRule(leaf["name"], leaf["status"], leaf["duration"] / 60.0, leaf["split"], + metrics.get("path_count", "?"), metrics.get("nonlinearity", "?"), top)) + for h in nl: + by_class[h.klass] = by_class.get(h.klass, 0) + h.pct + # a function is a "target" if it is a top nonlinear hotspot of this rule (pct-weighted) + for h in nl[:2]: + slot = fn_agg.setdefault(h.function, [0, h.klass, h.location, 0]) + slot[0] += h.pct + slot[3] += 1 + if h.location and not slot[2]: + slot[2] = h.location + + by_function = sorted( + (FunctionRollup(fn, v[1], v[2], v[0], v[3]) for fn, v in fn_agg.items()), + key=lambda f: f.nl_pct_sum, reverse=True) + return ProfileReport(job=job_url, cut=cut, spec=spec, n_rules=len(rules), + slow=slow, by_class=by_class, by_function=by_function) + + +def profile_jobs(job_urls: list[str], *, min_seconds: int = _DEFAULT_MIN_SECONDS, + cut: str | None = None) -> list[ProfileReport]: + """Profile several jobs (e.g. every component of one autoprover run) with a shared POU connection.""" + if not job_urls: + return [] + from .sources import _aiss_env_for + _aiss_env_for(job_urls[0]) + from prover_output_utility import ProverOutputAPI + api = ProverOutputAPI(use_local=False) + return [profile_job(u, min_seconds=min_seconds, cut=cut, api=api) for u in job_urls] + + +def aggregate_by_class(reports: list[ProfileReport]) -> dict: + """Sum the by-class nonlinear-op attribution across reports — the one-line 'where does the time go'.""" + total: dict = {} + for r in reports: + for k, v in r.by_class.items(): + total[k] = total.get(k, 0) + v + return dict(sorted(total.items(), key=lambda x: -x[1])) + + +def main(argv: list[str] | None = None) -> int: + import argparse + import json + p = argparse.ArgumentParser( + prog="difficulty-profile", + description="From completed prover run(s), attribute slow-rule prover time to source functions " + "and classify each as cut / library / external / cvl-model.") + p.add_argument("jobs", nargs="+", help="prover job URL(s) or hash(es) — e.g. every component of a run.") + p.add_argument("--min-minutes", type=float, default=5.0, help="slow-rule threshold (default 5).") + p.add_argument("--cut", default=None, help="override the contract under test (else from treeViewStatus).") + p.add_argument("--json", action="store_true", help="emit JSON instead of text.") + a = p.parse_args(argv) + reports = profile_jobs(a.jobs, min_seconds=int(a.min_minutes * 60), cut=a.cut) + if a.json: + print(json.dumps({"reports": [r.to_dict() for r in reports], + "aggregate_by_class": aggregate_by_class(reports)}, indent=2)) + else: + for r in reports: + print(r.format()) + print() + agg = aggregate_by_class(reports) + print("### aggregate — where the nonlinear ops go across all jobs (by class)") + for k, v in agg.items(): + print(f" {v:6d}% {k}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/summarization_detector/schema.py b/summarization_detector/schema.py new file mode 100644 index 00000000..6cc768d3 --- /dev/null +++ b/summarization_detector/schema.py @@ -0,0 +1,39 @@ +"""Wire schema for the detector's serialized output (``summarization_candidates.json``). + +TypedDict views of ``DetectionReport.to_dict`` — the single source of truth for consumers (autosetup +writes the file, composer renders it into the CVL-generation prompt). Kept import-light (typing only) so a +consumer can type against the schema without pulling in the detector's analysis code. + +These mirror the runtime dataclasses in ``detect.py`` (``Candidate`` / ``Boundary``); keep them in step. +""" + +from typing import NotRequired, TypedDict + + +class HostileBoundary(TypedDict): + """A caller (or shared inlined primitive) offered as an alternative place to summarize a candidate. + Serialized form of ``detect.Boundary`` — all fields are always present.""" + function: str + hops: int + signature: str + mutating: bool + direction: str + shared: int + + +class HostileCandidate(TypedDict): + """One prover-hostile summarization target. Serialized form of ``detect.Candidate``: + ``function``/``signals``/``score``/``evidence`` are always present; every other field is omitted when + it is at its default (see ``DetectionReport.to_dict``), so consumers must treat them as optional.""" + function: str + signals: list[str] + score: float + evidence: str + file: NotRequired[str] + line: NotRequired[int | None] + signature: NotRequired[str] + mutating: NotRequired[bool | None] + reaching_count: NotRequired[int] + summarizable: NotRequired[bool] + candidate_summary: NotRequired[str] + boundaries: NotRequired[list[HostileBoundary]] diff --git a/summarization_detector/sources.py b/summarization_detector/sources.py new file mode 100644 index 00000000..8aba71b5 --- /dev/null +++ b/summarization_detector/sources.py @@ -0,0 +1,167 @@ +"""Turn a prover-run URL into everything the detector needs, so `--url` is the only input: + + fetch the job SOURCES -> find its `run.conf` -> derive the main contract (the conf's `verify` field) + -> generate the AST (`certoraRun --compilation_steps_only --dump_asts`) -> best-effort the external call + graph -> run `detect` with the difficulty report the URL also provides. + +Live I/O (POU fetch + certoraRun), kept out of `detect.py`'s pure analysis core. Reuses POU +(`ProverOutputAPI.fetch_job_sources`) and `detect.ensure_ast`. vaas-dev URLs need `AISS_ENV=dev` — set +here from the host so the caller needn't. +""" +import json +import os +import tempfile +import time +from pathlib import Path +from typing import Callable, TypeVar + +from .detect import DetectionReport, detect, ensure_ast + +_T = TypeVar("_T") + +_TRANSIENT = ("connection reset", "connection aborted", "timed out", "temporarily unavailable", + "read timed out", "remotedisconnected", "max retries") + + +def _is_transient(e: Exception) -> bool: + return any(s in str(e).lower() for s in _TRANSIENT) + + +def _retry_transient(fn: Callable[[], _T], tries: int = 4) -> _T: + """Call fn, retrying only TRANSIENT network errors (the intermittent vaas-dev/prover connection + resets) with linear backoff; a non-transient error (auth, 404) is raised immediately.""" + for attempt in range(tries): + try: + return fn() + except Exception as e: + if attempt == tries - 1 or not _is_transient(e): + raise + time.sleep(2 * (attempt + 1)) + raise RuntimeError("unreachable: retry loop exhausted without return or raise") + + +def _aiss_env_for(url: str) -> None: + """vaas-dev jobs require AISS_ENV=dev for POU auth; set it from the URL host if unset.""" + if "vaas-dev" in url: + os.environ.setdefault("AISS_ENV", "dev") + + +def fetch_sources(url: str, dest: str | Path) -> Path: + """Fetch a job's `.certora_sources` tree to `dest` (POU), retrying transient resets. Returns the root.""" + _aiss_env_for(url) + from prover_output_utility import ProverOutputAPI + dest = Path(dest) + dest.mkdir(parents=True, exist_ok=True) + api = ProverOutputAPI(use_local=False) + return Path(_retry_transient(lambda: api.fetch_job_sources(url, dest))) + + +def find_run_conf(sources_dir: str | Path) -> Path: + """Locate the job's run conf in a fetched sources tree — the canonical + `inputs/.certora_sources/run.conf`, else the shallowest `run.conf`/`*.conf` outside lib/ and the + `.certora_internal` machinery.""" + root = Path(sources_dir) + canonical = root / "inputs" / ".certora_sources" / "run.conf" + if canonical.exists(): + return canonical + for pattern in ("run.conf", "*.conf"): + cands = [p for p in root.rglob(pattern) + if "/lib/" not in str(p) and "/.certora_internal/" not in str(p)] + if cands: + return min(cands, key=lambda p: len(p.parts)) + raise FileNotFoundError(f"no run.conf found under {root}") + + +def cut_from_conf(conf_path: str | Path) -> str: + """The main (verified) contract — the part before ':' in the conf's `verify` field + (`"Router:certora/specs/x.spec"` -> `"Router"`), falling back to the first `parametric_contracts`.""" + conf = json.loads(Path(conf_path).read_text()) + verify = conf.get("verify") or "" + if isinstance(verify, str) and ":" in verify: + return verify.split(":", 1)[0] + parametric = conf.get("parametric_contracts") or [] + return parametric[0] if parametric else "" + + +def find_external_call_graph(sources_dir: str | Path) -> Path | None: + """The prover's `externalCallGraph.json` if a local tree already carries one (e.g. a local prover run's + `Reports/`). None otherwise. For a job URL use `fetch_external_call_graph` — it's a Reports artifact, + NOT a source file, so it isn't in `fetch_job_sources` output.""" + hits = list(Path(sources_dir).rglob("externalCallGraph.json")) + return hits[0] if hits else None + + +def fetch_external_call_graph(url: str, dest: str | Path) -> Path | None: + """Fetch just `Reports/externalCallGraph.json` — the prover writes it under `Reports/`, not into + `.certora_sources`, so the source-files fetch misses it. Uses POU's single-file endpoint + (`fetch_output_file` -> `/f/`), NOT the whole output tarball. Returns the written path, or + None when the run produced none (a prover build without the collector) or on any fetch error. + NB: needs an up-to-date POU (the ProverCLI repo) — older installs lack `fetch_output_file`.""" + _aiss_env_for(url) + try: + from prover_output_utility import ProverOutputAPI + api = ProverOutputAPI(use_local=False) + # fetch_output_file needs a current POU (ProverCLI); older installs lack it -> AttributeError -> + # caught below -> no ecg (graceful). type: ignore because the installed stub may be stale. + content = _retry_transient( + lambda: api.fetch_output_file(url, "externalCallGraph.json")) # type: ignore[attr-defined] + if not content: + return None + out = Path(dest) / "externalCallGraph.json" + out.write_text(content if isinstance(content, str) else json.dumps(content)) + return out + except Exception: + return None + + +def fetch_surviving_graphs(url: str) -> list[dict]: + """Fetch a run's postOptimize SurvivingCallGraph dumps and return them PARSED. Reads the manifest + `survivingCallGraph_map.json` (POU single-file endpoint), then each rule's postOptimize file. Empty + list when the run produced none (a prover build without the collector) or on any fetch error. The + detector derives both signal 4 (surviving hostile primitives) and the signal-2 reachability gate from + these. NB: needs an up-to-date POU (the ProverCLI repo).""" + _aiss_env_for(url) + graphs: list[dict] = [] + try: + from prover_output_utility import ProverOutputAPI + api = ProverOutputAPI(use_local=False) + # fetch_output_file needs a current POU; older installs lack it -> AttributeError -> caught -> []. + raw = _retry_transient( + lambda: api.fetch_output_file(url, "survivingCallGraph_map.json")) # type: ignore[attr-defined] + if not raw: + return [] + manifest = json.loads(raw) if isinstance(raw, str) else raw + except Exception: + return [] + for files in manifest.values(): + for fn in files: + if "postOptimize" not in fn: + continue + try: + content = _retry_transient( + lambda fn=fn: api.fetch_output_file(url, fn)) # type: ignore[attr-defined] + if content: + graphs.append(json.loads(content) if isinstance(content, str) else content) + except Exception: + continue + return graphs + + +def detect_url(url: str, *, work_dir: str | Path | None = None, solc_dir: str | Path | None = None, + cut: str | None = None, external_call_graph: str | Path | None = None, + include_dependencies: bool = False) -> DetectionReport: + """The one-input entry: from a prover-run URL, fetch + derive everything and run the detector. `cut` + and `external_call_graph` override what would otherwise be derived/fetched. `work_dir` (default: a + temp dir) holds the fetched sources + generated AST.""" + work = Path(work_dir) if work_dir else Path(tempfile.mkdtemp(prefix="detect-")) + src = fetch_sources(url, work / "sources") + conf = find_run_conf(src) + cut = cut or cut_from_conf(conf) + if not cut: + raise ValueError(f"could not derive the main contract from {conf} — pass cut=") + ast = ensure_ast(conf=conf, solc_dir=solc_dir) + ecg = external_call_graph or fetch_external_call_graph(url, work) # Reports artifact — from the tarball + surviving_graphs = fetch_surviving_graphs(url) # signal 4 + the signal-2 gate + return detect(url, ast_path=ast, cut=cut, external_call_graph=ecg, + surviving_graphs=surviving_graphs, include_dependencies=include_dependencies, + sources_root=conf.parent) diff --git a/summarization_detector/tests/__init__.py b/summarization_detector/tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/summarization_detector/tests/test_detect.py b/summarization_detector/tests/test_detect.py new file mode 100644 index 00000000..7a7c8b3c --- /dev/null +++ b/summarization_detector/tests/test_detect.py @@ -0,0 +1,509 @@ +"""Summarization-target detector (detect.py). Fast, offline, no prover/scene: + - scan_ast: a synthetic solc-AST fixture exercises span-attribution, the `abi.`-base guard (a non-abi + `.encode()` is NOT hashing), source-file dedup, and the lib/ dependency flag. + - detect_from: the signal fusion (per-function candidates, ranked). +No real `.asts.json` is needed — scan_ast streams any JSON of the documented shape.""" +import json +import tempfile +from pathlib import Path + +from summarization_detector.detect import ( + scan_ast, detect_from, HashSignal, reachable_from_main, cone_weights, _touch_missing_imported_specs, + _function_locations, + _survives, _caller_boundaries, _expressible_typename, +) +from summarization_detector.difficulty import DifficultyReport, Hotspot + + +def _node(nt, off, length, **kw): + return {"nodeType": nt, "src": f"{off}:{length}:1", **kw} + + +def _args(*specs): + """Each spec is a type string, or a (type, ref_id) tuple attaching a `referencedDeclaration` — used to + model an argument that reads a function PARAMETER (the user-input test for dynamic-length hashing).""" + out = [] + for s in specs: + if isinstance(s, tuple): + out.append({"typeDescriptions": {"typeString": s[0]}, "referencedDeclaration": s[1]}) + else: + out.append({"typeDescriptions": {"typeString": s}}) + return out + + +def _fdef(off, length, name, *, vis="internal", mut="pure", param_ids=()): + return _node("FunctionDefinition", off, length, name=name, stateMutability=mut, visibility=vis, + parameters={"parameters": [{"id": i} for i in param_ids]}) + + +def _ident_call(off, name, *type_strings): + return _node("FunctionCall", off, 8, arguments=_args(*type_strings), + expression={"nodeType": "Identifier", "name": name}) + + +def _member_call(off, member, base, *type_strings): + return _node("FunctionCall", off, 8, arguments=_args(*type_strings), + expression={"nodeType": "MemberAccess", "memberName": member, + "expression": {"nodeType": "Identifier", "name": base}}) + + +def _write_ast(tmp, files: dict) -> Path: + """files: {abs_path: {node_id: node}} -> a .asts.json keyed by one compilation unit per file.""" + doc = {abs_path: {abs_path: nodes} for abs_path, nodes in files.items()} + p = Path(tmp) / "x.asts.json" + p.write_text(json.dumps(doc)) + return p + + +def test_scan_ast_attributes_guards_and_flags(): + nodes = { + "1": _node("ContractDefinition", 0, 200, name="Lib", contractKind="library"), + "2": _node("FunctionDefinition", 10, 90, name="hashIt", stateMutability="pure", visibility="internal"), + "3": _ident_call(50, "keccak256"), # in hashIt -> flagged + "4": _member_call(60, "encodePacked", "abi"), # in hashIt -> flagged + "5": _node("FunctionDefinition", 110, 80, name="plain", stateMutability="view", visibility="public"), + "6": _member_call(150, "encode", "myCodec"), # NON-abi .encode in plain -> NOT flagged + "7": _ident_call(195, "keccak256"), # file-scope (outside any fn) -> skipped + } + with tempfile.TemporaryDirectory() as d: + sigs = scan_ast(_write_ast(d, {"src/A.sol": nodes})) + assert len(sigs) == 1 + s = sigs[0] + assert s.function == "Lib.hashIt" and s.contract == "Lib" and s.name == "hashIt" + assert s.mutability == "pure" and s.visibility == "internal" + assert s.patterns == ("abi.encodePacked", "keccak256") # sorted; the non-abi encode is excluded + assert s.is_dependency is False + + +def test_scan_ast_requires_hash_trigger_and_classifies_length(): + """Only a function that calls a HASH builtin is a candidate — an encoder-only function (serialization + or calldata via encodeWithSelector) is NOT. Length class: a hash over dynamic data (bytes/string) is + dynamic-input; over fixed-size fields it is fixed.""" + nodes = { + "1": _node("ContractDefinition", 0, 400, name="H", contractKind="contract"), + # fixed-size digest: keccak(abi.encode(uint256, address)) + "2": _fdef(10, 80, "hashFixed"), + "3": _ident_call(30, "keccak256", "bytes memory"), + "4": _member_call(40, "encode", "abi", "uint256", "address"), + # dynamic digest over a PARAMETER (id 900): keccak(abi.encodePacked(bytes param)) + "5": _fdef(100, 80, "hashDyn", param_ids=(900,)), + "6": _ident_call(120, "keccak256", "bytes memory"), + "7": _member_call(130, "encodePacked", "abi", ("bytes memory", 900)), + # serialization only (abi.encode, no hash) -> dropped + "8": _fdef(200, 80, "serialize"), + "9": _member_call(220, "encode", "abi", "uint256"), + # calldata construction (encodeWithSelector is not even a tracked encoder) -> dropped + "10": _fdef(300, 80, "callData", vis="internal", mut="view"), + "11": _member_call(320, "encodeWithSelector", "abi", "bytes4", "address"), + } + with tempfile.TemporaryDirectory() as d: + sigs = {s.function: s for s in scan_ast(_write_ast(d, {"src/H.sol": nodes}))} + assert set(sigs) == {"H.hashFixed", "H.hashDyn"} # serialize + callData dropped (no hash trigger) + assert sigs["H.hashFixed"].dynamic_input is False + assert sigs["H.hashDyn"].dynamic_input is True # dynamic operand reads a parameter + + +def test_scan_ast_constant_dynamic_typed_input_is_not_dynamic(): + """A hash over a dynamic-TYPED but non-parameter operand (a constant/immutable/local, e.g. + `keccak256(bytes(name))` for a fixed EIP-712 name) is NOT dynamic-input — it is bounded/cheap.""" + nodes = { + "1": _node("ContractDefinition", 0, 200, name="E", contractKind="contract"), + "2": _fdef(10, 90, "nameHash", vis="internal", mut="view"), # no params + "3": _ident_call(40, "keccak256", "bytes memory"), # arg reads a LOCAL, not a param + } + with tempfile.TemporaryDirectory() as d: + sigs = {s.function: s for s in scan_ast(_write_ast(d, {"src/E.sol": nodes}))} + assert set(sigs) == {"E.nameHash"} + assert sigs["E.nameHash"].dynamic_input is False # constant-valued -> not expensive + + +def test_scan_ast_drops_ecrecover_only(): + """`ecrecover`-only functions are dropped (a recovered address has no over-approximable property); a + function that also hashes is kept.""" + nodes = { + "1": _node("ContractDefinition", 0, 300, name="S", contractKind="contract"), + "2": _fdef(10, 90, "recover", param_ids=(900,)), # only ecrecover -> dropped + "3": _ident_call(40, "ecrecover", "bytes32", "uint8", "bytes32", "bytes32"), + "4": _fdef(110, 90, "digestAndRecover", param_ids=(901,)), # keccak + ecrecover -> kept + "5": _ident_call(140, "keccak256", ("bytes memory", 901)), + "6": _ident_call(160, "ecrecover", "bytes32", "uint8", "bytes32", "bytes32"), + } + with tempfile.TemporaryDirectory() as d: + sigs = {s.function: s for s in scan_ast(_write_ast(d, {"src/S.sol": nodes}))} + assert set(sigs) == {"S.digestAndRecover"} # recover (ecrecover-only) dropped + + +def test_scan_ast_dedups_and_flags_dependency(): + """A source file recurs under many compilation units — processed ONCE. A lib/ path is a dependency.""" + lib_nodes = { + "1": _node("ContractDefinition", 0, 100, name="ERC20", contractKind="contract"), + "2": _node("FunctionDefinition", 10, 80, name="permit", stateMutability="nonpayable", visibility="public"), + "3": _ident_call(40, "keccak256"), + } + # same abs path appears under two compilation units (two top-level rel keys) + p = Path(tempfile.mkdtemp()) / "x.asts.json" + p.write_text(json.dumps({ + "unitA": {"lib/solady/src/tokens/ERC20.sol": lib_nodes}, + "unitB": {"lib/solady/src/tokens/ERC20.sol": lib_nodes}, + })) + sigs = scan_ast(p) + assert len(sigs) == 1 and sigs[0].function == "ERC20.permit" # deduped, not counted twice + assert sigs[0].is_dependency is True # under lib/ + + +def test_detect_from_fuses_and_classifies(): + """Signals fuse per function: a cross-contract hotspot gets `external`+`nonlinear`; the CUT's own + INLINED internal math gets `nonlinear`; a hasher gets `hashing`. The CUT's own EXTERNAL method (an + unmarked `.m` — a rule subject) is dropped. Dependency hashing is dropped by default.""" + diff = DifficultyReport(hotspots=[ + Hotspot("Oracle.getPrice", 40, "O.sol:10"), # external (not the CUT) + Hotspot("(internal) C.mulThing", 55, "C.sol:5"), # the CUT's own inlined internal math + Hotspot("C.borrow", 70, "C.sol:9"), # the CUT's own EXTERNAL method (rule subject) + ]) + hs = [ + HashSignal("C.hashId", "C", "hashId", "pure", "internal", ("keccak256",), "src/C.sol", False), + HashSignal("Dep.enc", "Dep", "enc", "pure", "internal", ("abi.encode",), "lib/x/Dep.sol", True), + ] + rep = detect_from(hs, diff, cut="C") + by = {c.function: c for c in rep.candidates} + + assert "external" in by["Oracle.getPrice"].signals + assert by["C.mulThing"].signals == ("nonlinear",) # CUT's own internal math, not external + assert "C.borrow" not in by # CUT's external entry method dropped + assert "hashing" in by["C.hashId"].signals + assert "Dep.enc" not in by # dependency hashing filtered by default + assert rep.candidates == sorted(rep.candidates, key=lambda c: (-c.score, c.function)) # ranked + + +def test_detect_from_drops_cvl_ghost_hotspots(): + # a CVL/ghost hotspot is an ALREADY-APPLIED summary — must never be a candidate (it IS the summary) + diff = DifficultyReport(hotspots=[ + Hotspot("CVL/Ghost Function 'mulDivDownSummary(x,y,denominator)'", 86, ""), # already a summary + Hotspot("AmountConverter.getExpectedOut", 20, "contracts/AmountConverter.sol:134"), # external callee + ]) + rep = detect_from([], diff, cut="Stonks") + fns = {c.function for c in rep.candidates} + assert "AmountConverter.getExpectedOut" in fns # real math kept + assert not any("Ghost" in f or "CVL" in f for f in fns) # ghost summary dropped + + +def test_detect_from_keeps_cut_internal_math_but_drops_cut_external_method(): + # (internal)-marked CUT hotspot = an inlined internal fn (summarizable); an unmarked CUT hotspot = the + # CUT's own external method (a rule subject) and is dropped. + diff = DifficultyReport(hotspots=[ + Hotspot("(internal) Stonks.estimateTradeOutput", 20, "S.sol:387"), + Hotspot("Stonks.swap", 80, "S.sol:40"), + ]) + by = {c.function: c for c in detect_from([], diff, cut="Stonks").candidates} + assert by["Stonks.estimateTradeOutput"].signals == ("nonlinear",) # internal CUT math kept, not external + assert "Stonks.swap" not in by # CUT's external method dropped + + +def test_detect_from_nonlinear_floor_and_internal_dedup(): + from summarization_detector.detect import NONLINEAR_MIN_PCT + diff = DifficultyReport(hotspots=[ + Hotspot("Hub.previewShares", NONLINEAR_MIN_PCT, "Hub.sol:471"), # exactly at the floor -> kept + Hotspot("Hub.getIndex", NONLINEAR_MIN_PCT - 1, "Hub.sol:508"), # below the floor -> dropped + Hotspot("(internal) Hub.calcRay", 27, "AssetLogic.sol:153"), # kept (top instance of calcRay) + Hotspot("(internal) Spoke.calcRay", 25, "Spoke.sol:639"), # same bare name, lower -> deduped + ]) + by = {c.function: c for c in detect_from([], diff, cut="Spoke").candidates} + assert "Hub.previewShares" in by and "Hub.getIndex" not in by + assert "Hub.calcRay" in by and "Spoke.calcRay" not in by # caller-attribution dedup by bare name + + +def _fndef(off, length, name, vis="internal"): + return _node("FunctionDefinition", off, length, name=name, stateMutability="pure", visibility=vis) + + +def _call_ref(off, ref): + return _node("FunctionCall", off, 8, expression={"nodeType": "Identifier", "name": "x", "referencedDeclaration": ref}) + + +def _reach_fixture(tmp, orphan_name="orphan"): + """Main.entry (external) internally calls `reached`; `` is defined but uncalled. Both + reached and orphan hash (keccak) so scan_ast flags them; reachability decides which survives.""" + nodes = { + "1": _node("ContractDefinition", 0, 400, name="Main", contractKind="contract"), + "10": _fndef(10, 90, "entry", vis="external"), + "11": _call_ref(50, 20), # entry -> reached (referencedDeclaration=20) + "20": _fndef(110, 90, "reached"), + "21": _ident_call(150, "keccak256", "bytes memory"), + "22": _member_call(155, "encodePacked", "abi", "bytes"), + "30": _fndef(210, 90, orphan_name), + "31": _ident_call(250, "keccak256", "bytes memory"), + "32": _member_call(255, "encodePacked", "abi", "bytes"), + } + return _write_ast(tmp, {"src/Main.sol": nodes}) + + +def test_survives_matches_free_functions_by_bare_name(): + # The prover attributes a free (file-level) function to an arbitrary host in the surviving set. + surviving = {"Ownable.computeBaseHash", "CombinatorialModule.getConditionId", "Router.convert"} + bare = {n.rpartition(".")[2] for n in surviving} + # host-less candidate (a free function) matches by bare name despite the mis-attributed host + assert _survives("computeBaseHash", surviving, bare) + # hosted candidate matches only exactly — a genuinely-unreachable CTHelpers.getConditionId is NOT + # resurrected by the same-named method on another contract + assert not _survives("CTHelpers.getConditionId", surviving, bare) + assert _survives("CombinatorialModule.getConditionId", surviving, bare) + # host-less candidate with no bare-name match stays out + assert not _survives("notReachable", surviving, bare) + + +def _elem(name): + return {"nodeType": "ElementaryTypeName", "name": name} + + +def _arr(base): + return {"nodeType": "ArrayTypeName", "baseType": base} + + +def _udt(ref): + return {"nodeType": "UserDefinedTypeName", "referencedDeclaration": ref} + + +def test_expressible_typename_recurses_arrays_and_structs(): + # value types / fixed bytes expressible; dynamic bytes/string not + assert _expressible_typename(_elem("uint256"), {}) and _expressible_typename(_elem("bytes32"), {}) + assert not _expressible_typename(_elem("bytes"), {}) and not _expressible_typename(_elem("string"), {}) + # NESTING the flat check missed: bytes[] and struct-with-a-bytes-field are NOT expressible + assert _expressible_typename(_arr(_elem("uint256")), {}) + assert not _expressible_typename(_arr(_elem("bytes")), {}) # bytes[] + merged = { + "10": {"nodeType": "UserDefinedValueTypeDefinition"}, # a UDVT (e.g. PositionId) + "20": {"nodeType": "StructDefinition", "members": [{"typeName": _elem("bytes")}]}, + "30": {"nodeType": "StructDefinition", "members": [{"typeName": _elem("uint256")}]}, + "50": {"nodeType": "StructDefinition", # struct WITH a mapping field + "members": [{"typeName": {"nodeType": "Mapping"}}, {"typeName": _elem("uint256")}]}, + } + assert _expressible_typename(_udt(10), merged) # UDVT -> value-like + assert _expressible_typename(_arr(_udt(10)), merged) # PositionId[] -> expressible + assert not _expressible_typename(_udt(20), merged) # struct WITH a bytes field + assert _expressible_typename(_udt(30), merged) # struct of value types + assert not _expressible_typename({"nodeType": "Mapping"}, merged) # mapping + assert not _expressible_typename(_udt(50), merged) # mapping nested in a struct + + +def test_caller_boundaries_keeps_only_expressible_internal_callers(): + # leaf <- encodeFromData (opaque, dropped) <- mutClose (clean+mutating) <- pureFar (clean+pure); + # a public entry point that also calls the leaf is dropped (the rule's subject, not a boundary). + edges = { + "L.encodeFromData": {"computeBaseHash"}, + "M.mutClose": {"L.encodeFromData"}, + "M.pureFar": {"M.mutClose"}, + "C.borrow": {"computeBaseHash"}, + } + sigs = { # (signature, expressible, mutating, is_external) + "L.encodeFromData": ("encodeFromData(uint256, bytes) -> C", False, False, False), # opaque -> dropped + "M.mutClose": ("mutClose(PositionId[]) -> C", True, True, False), # clean but mutating + "M.pureFar": ("pureFar(PositionId[]) -> C", True, False, False), # clean + pure + "C.borrow": ("borrow(uint256) -> uint256", True, True, True), # external -> dropped + } + reach = {"computeBaseHash", "L.encodeFromData", "M.mutClose", "M.pureFar", "C.borrow"} + b = _caller_boundaries(edges, sigs, "computeBaseHash", reach) + # opaque + external-entry both dropped; pure+clean wins over mutating-clean (view-preference beats distance) + assert [x.function for x in b] == ["M.pureFar", "M.mutClose"] + assert not b[0].mutating and b[1].mutating + + +def test_caller_boundaries_filters_to_reachable(): + edges = {"C.caller": {"computeBaseHash"}} + sigs = {"C.caller": ("caller(uint256) -> bytes32", True, False, False)} # expressible, pure, internal + assert _caller_boundaries(edges, sigs, "computeBaseHash", reachable={"computeBaseHash"}) == [] # caller unreached + assert _caller_boundaries(edges, sigs, "computeBaseHash", reachable=None) # no filter -> kept + + +def test_boundary_format_flags_mutating_vs_pure(): + from summarization_detector.detect import DetectionReport, Candidate, Boundary + # only expressible, internal boundaries reach the report; mutating is the last remaining caveat + rep = DetectionReport(candidates=[Candidate( + "leaf", ("hashing",), 10.0, "ev", + boundaries=[ + Boundary("A.pureView", 1, "pureView(address) -> b32", mutating=False), + Boundary("A.mutClose", 2, "mutClose(address) -> b32", mutating=True), + ])]) + lines = rep.format().splitlines() + pure_line = next(ln for ln in lines if "pureView" in ln) + mut_line = next(ln for ln in lines if "mutClose" in ln) + assert "summarizable here" in pure_line + assert "state-changing" in mut_line + + +def test_descend_to_prims_finds_shared_nonlinear_primitive(): + from summarization_detector.detect import _descend_to_prims, _is_nonlinear_prim + assert _is_nonlinear_prim("MathLib.mulDivDown") and not _is_nonlinear_prim("V.previewDeposit") + edges = { # both methods reach MathLib.mulDivDown + "V.previewDeposit": {"V.accrueInterestView", "MathLib.mulDivDown"}, + "V.accrueInterestView": {"MathLib.mulDivDown"}, + "V.deposit": {"V.previewDeposit"}, + } + assert _descend_to_prims(edges, "V.previewDeposit", None) == {"MathLib.mulDivDown": 1} + assert _descend_to_prims(edges, "V.deposit", None) == {"MathLib.mulDivDown": 2} # transitive, min depth + # a primitive not in the reachable set is dropped (never suggest dead code) + assert _descend_to_prims(edges, "V.previewDeposit", reachable={"V.previewDeposit"}) == {} + + +def test_boundary_down_direction_renders_shared_count(): + from summarization_detector.detect import DetectionReport, Candidate, Boundary + rep = DetectionReport(candidates=[Candidate( + "V.previewDeposit", ("nonlinear",), 100.0, "ev", + boundaries=[Boundary("MathLib.mulDivDown", 2, "mulDivDown(uint256, uint256, uint256) -> uint256", + mutating=False, direction="down", shared=8)])]) + line = next(ln for ln in rep.format().splitlines() if "mulDivDown" in ln) + assert "↓" in line and "shared ×8" in line and "summarizable here" in line + + +def test_reachable_from_main_internal_edges(): + """BFS from Main's external entry over AST referencedDeclaration edges: `reached` is in, `orphan` out. + Gating scan_ast on the reachable set drops the scene-unreachable hashing candidate.""" + with tempfile.TemporaryDirectory() as d: + ast = _reach_fixture(d) + reach = reachable_from_main(ast, "Main") + assert "Main.reached" in reach and "Main.orphan" not in reach + cands = {h.function for h in scan_ast(ast)} + assert cands == {"Main.reached", "Main.orphan"} # scan flags both + gated = {c for c in cands if c in reach} + assert gated == {"Main.reached"} # reachability drops the orphan + + +def test_cone_weights_sums_consumer_code_size(): + """Cone-of-influence heuristic: a function's weight is the total body size of its transitive consumers + (reachable callers). `Main.reached` is consumed only by `Main.entry` (body size 90); a root has none; + an unreachable function is not weighted.""" + with tempfile.TemporaryDirectory() as d: + ast = _reach_fixture(d) + cone = cone_weights(ast, "Main") + assert cone["Main.reached"] == 90 # consumed by Main.entry (span 10..100 -> 90 bytes) + assert cone["Main.entry"] == 0 # a root — nothing consumes it + assert "Main.orphan" not in cone # unreachable -> not weighted + + +def test_reachable_from_main_external_dispatch_edge(): + """A dispatch call site (no resolved contract, selector `merge`) in externalCallGraph.json makes the + otherwise-unreachable `Main.merge` reachable via selector-name matching.""" + with tempfile.TemporaryDirectory() as d: + ast = _reach_fixture(d, orphan_name="merge") + assert "Main.merge" not in reachable_from_main(ast, "Main") # unreachable without external edges + ecg = Path(d) / "externalCallGraph.json" + ecg.write_text(json.dumps({ + "Main": [{"caller": "entry(uint256)", "selectors": [{"kind": "sighash", "signature": "merge(uint256)"}], + "targets": [{"resolution": "symbolicOutput"}]}] # dispatch: no contract + })) + reach = reachable_from_main(ast, "Main", ecg) + assert "Main.merge" in reach # selector-matched dispatch edge + + +def test_touch_missing_imported_specs_recreates_empty_placeholders(): + """The empty-file-skip workaround: parse certoraRun's missing-import error and recreate the (empty) + imported spec so a re-run resolves it. Only acts on the missing-import error, and never clobbers.""" + with tempfile.TemporaryDirectory() as d: + missing = Path(d) / "certora" / "specs" / "summaries" / "C_call_resolution.spec" + err = (f'In {d}/certora/specs/sanity-C.spec, the following import declarations do not import ' + f'existing .spec files:\n2:2:"{missing}"\n') + created = _touch_missing_imported_specs("", err) + assert created == [missing] + assert missing.exists() and missing.read_text() == "" # empty placeholder, matches the real file + assert _touch_missing_imported_specs("", err) == [] # already exists -> no-op, no clobber + assert _touch_missing_imported_specs("some unrelated compile error") == [] # only the import error + + +def test_detect_from_includes_dependencies_when_asked(): + hs = [HashSignal("Dep.enc", "Dep", "enc", "pure", "internal", ("abi.encode",), "lib/x/Dep.sol", True)] + rep = detect_from(hs, DifficultyReport(), cut="C", include_dependencies=True) + assert any(c.function == "Dep.enc" for c in rep.candidates) + + +def test_function_locations_file_and_line(): + """file always (the AST node group's path); line resolved from the source byte-offset when + `sources_root` is given and the file is readable, else None.""" + # "foo" starts at byte 12 = after "line1\n" (6) + "line2\n" (6) -> line 3 + nodes = { + "1": _node("ContractDefinition", 0, 200, name="Lib"), + "2": _node("FunctionDefinition", 12, 50, name="foo"), + } + with tempfile.TemporaryDirectory() as d: + ast = _write_ast(d, {"src/A.sol": nodes}) + (Path(d) / "src").mkdir() + (Path(d) / "src/A.sol").write_text("line1\nline2\nfunction foo() {}\n") + with_src = _function_locations(ast, sources_root=d) + no_src = _function_locations(ast) + assert with_src["Lib.foo"] == ("src/A.sol", 3) + assert no_src["Lib.foo"] == ("src/A.sol", None) # file only, line unresolved without sources + + +def test_detect_from_caps_each_category_and_reports_dropped(): + from summarization_detector.detect import detect_from, HashSignal, MAX_PER_CATEGORY + from summarization_detector.difficulty import DifficultyReport + cap = MAX_PER_CATEGORY["hashing"] + # more hashers than the hashing cap -> that category is bounded to its cap and `dropped` records the rest + hs = [HashSignal(function=f"C.h{i}", contract="C", name=f"h{i}", mutability="view", + visibility="internal", patterns=("keccak256",), file="src/C.sol", + is_dependency=False, dynamic_input=True) + for i in range(cap + 5)] + rep = detect_from(hs, DifficultyReport(), cut="C") + hashing = [c for c in rep.candidates if "hashing" in c.signals] + assert len(hashing) == cap # hashing category capped + assert rep.dropped == 5 # the 5 over the cap are dropped + + +def test_surviving_score_is_flat_regardless_of_reach(): + from summarization_detector.detect import detect_from, SURVIVING_SCORE + from summarization_detector.difficulty import DifficultyReport + # two primitives, wildly different reach -> same (flat) score; reach rides along as reaching_count only + surviving = { + "L.wide(uint256)": {"category": "symbolic-exp", "reason": "", "reaching_methods": [f"m{i}" for i in range(20)], + "summarizable": True, "candidate_summary": ""}, + "L.narrow(uint256)": {"category": "bitwise-scan", "reason": "", "reaching_methods": ["only"], + "summarizable": True, "candidate_summary": ""}, + } + rep = detect_from([], DifficultyReport(), cut="C", surviving=surviving) + by = {c.function: c for c in rep.candidates} + assert by["L.wide"].score == SURVIVING_SCORE and by["L.narrow"].score == SURVIVING_SCORE + assert by["L.wide"].reaching_count == 20 and by["L.narrow"].reaching_count == 1 + assert "reaches 20" in by["L.wide"].evidence # count still reported in evidence + + +def test_signatures_gives_signature_and_mutating_with_bare_fallback(): + from summarization_detector.detect import _signatures + merged = { + "1": {"nodeType": "ContractDefinition", "src": "0:200:1", "name": "AssetLogic"}, + "2": {"nodeType": "FunctionDefinition", "src": "10:80:1", "name": "calcRay", "stateMutability": "view", + "visibility": "internal", + "parameters": {"parameters": [{"typeDescriptions": {"typeString": "uint256"}}]}, + "returnParameters": {"parameters": [{"typeDescriptions": {"typeString": "uint256"}}]}}, + "3": {"nodeType": "ContractDefinition", "src": "300:200:1", "name": "Hub"}, + "4": {"nodeType": "FunctionDefinition", "src": "310:80:1", "name": "add", "stateMutability": "nonpayable", + "visibility": "external", + "parameters": {"parameters": [{"typeDescriptions": {"typeString": "uint256"}}]}, + "returnParameters": {"parameters": []}}, + } + sigs = _signatures(merged) # (signature, expressible, mutating, is_external) + assert sigs["AssetLogic.calcRay"][0] == "calcRay(uint256) -> uint256" and sigs["AssetLogic.calcRay"][2] is False + assert sigs["Hub.add"][0] == "add(uint256)" and sigs["Hub.add"][2] is True # nonpayable -> mutating + # the attach's bare-name fallback: a caller-attributed hotspot resolves by bare name + bare = {q.rpartition(".")[2]: v for q, v in sigs.items()} + assert bare["calcRay"][0] == "calcRay(uint256) -> uint256" and bare["calcRay"][2] is False + + +def test_function_locations_normalizes_absolute_ast_paths(): + # solc records some source-unit keys absolute (remapped imports); the stored `file` must still be + # project-relative (uniform with the relative-keyed ones), with the line read from the absolute path. + with tempfile.TemporaryDirectory() as d: + root = Path(d) / "inputs" / ".certora_sources" + absfile = root / "src" / "math" / "MathUtils.sol" + absfile.parent.mkdir(parents=True) + absfile.write_text("a\nb\nfunction uncheckedExp() {}\n") + doc = { + str(absfile): {str(absfile): { # ABSOLUTE source-unit key + "1": {"nodeType": "ContractDefinition", "src": "0:80:1", "name": "MathUtils"}, + "2": {"nodeType": "FunctionDefinition", "src": "4:20:1", "name": "uncheckedExp"}}}, + "src/dep/LibBit.sol": {"src/dep/LibBit.sol": { # already-relative key + "3": {"nodeType": "ContractDefinition", "src": "0:80:1", "name": "LibBit"}, + "4": {"nodeType": "FunctionDefinition", "src": "4:20:1", "name": "fls"}}}, + } + ap = Path(d) / "x.asts.json" + ap.write_text(json.dumps(doc)) + locs = _function_locations(ap, sources_root=root) + assert locs["MathUtils.uncheckedExp"] == ("src/math/MathUtils.sol", 3) # absolute key -> relative + assert locs["LibBit.fls"][0] == "src/dep/LibBit.sol" # relative key -> unchanged diff --git a/summarization_detector/tests/test_difficulty_profile.py b/summarization_detector/tests/test_difficulty_profile.py new file mode 100644 index 00000000..337cb94f --- /dev/null +++ b/summarization_detector/tests/test_difficulty_profile.py @@ -0,0 +1,57 @@ +"""Difficulty profiler (difficulty_profile.py) — the pure, offline parts: source classification +(cut / library / external / cvl-model) and the difficulty-tree parse + hotspot roll-up. The live POU +fetch (profile_job/profile_jobs) is exercised end-to-end against real jobs, not here.""" +from summarization_detector.difficulty_profile import ( + _classify, _parse_difficulty_tree, _iter_slow_leaves, +) + +CUT = "Vault" +SCENE = {"Vault", "OracleHarness", "TokenHarness"} + + +def test_classify_cut_external_library_cvl(): + assert _classify("Vault.computeAccountData", CUT, SCENE) == ("Vault", "cut") + # a linked scene contract that is NOT the CUT -> external dependency + assert _classify("OracleHarness.getPrice", CUT, SCENE)[1] == "external" + # not a scene contract and not the CUT -> an inlined library + assert _classify("ValuationLib.toValue", CUT, SCENE) == ("ValuationLib", "library") + # an applied CVL summary contributes nothing to model + assert _classify("CVL/Ghost Function 'cvlPrice(id)'", CUT, SCENE)[1] == "cvl-model" + + +def test_parse_tree_extracts_metrics_and_ranked_hotspots(): + tree = {"label": "verification phase", "children": [ + {"label": "path count", "value": "approx. 2^127"}, + {"label": "nonlinearity", "value": "nonlinear ops: 286\nmax polyn. degree: 6"}, + {"label": "nonlinearity hotspots", "children": [ + {"label": "function: Vault.withdraw", + "value": "contrib. to nonlinear ops: 68 %", + "jumpToDefinition": {"file": "src/Vault.sol", "start": {"line": 244}}}, + {"label": "function: OracleHarness.getPrice", + "value": "contrib. to nonlinear ops: 12 %", + "jumpToDefinition": {"file": "src/Oracle.sol", "start": {"line": 471}}}, + ]}, + ]} + metrics, hs = _parse_difficulty_tree(tree, CUT, SCENE) + assert metrics["path_count"] == "approx. 2^127" + assert "286" in metrics["nonlinearity"] + nl = sorted([h for h in hs if h.kind == "nl"], key=lambda h: h.pct, reverse=True) + assert (nl[0].function, nl[0].pct, nl[0].klass, nl[0].location) == ( + "Vault.withdraw", 68, "cut", "src/Vault.sol:244") + assert nl[1].klass == "external" and nl[1].pct == 12 + + +def test_iter_slow_leaves_recurses_children_and_thresholds(): + rules = [{"name": "inv", "status": "VIOLATED", "duration": 1862, "LiveCheckInfo": None, "children": [ + {"name": "Induction step", "status": "TIMEOUT", "duration": 1837, "LiveCheckInfo": None, "children": [ + {"name": "methodA", "status": "TIMEOUT", "duration": 1808, + "LiveCheckInfo": "rule_live_statistics_86.json", "children": []}, + {"name": "fast", "status": "VERIFIED", "duration": 9, + "LiveCheckInfo": "rule_live_statistics_75.json", "children": []}, + ]}, + ]}] + leaves = list(_iter_slow_leaves(rules, min_seconds=1200)) + # only the slow leaf with a live-stats file; the induction-scaffolding name is not prefixed in + names = [l["name"] for l in leaves] + assert len(leaves) == 1 and names[0].endswith("methodA") and "Induction" not in names[0] + assert leaves[0]["lci"] == "rule_live_statistics_86.json" diff --git a/summarization_detector/tests/test_sources.py b/summarization_detector/tests/test_sources.py new file mode 100644 index 00000000..cd206a6d --- /dev/null +++ b/summarization_detector/tests/test_sources.py @@ -0,0 +1,43 @@ +"""URL→inputs plumbing (sources.py) — the pure, offline parts: deriving the main contract from a conf's +`verify` field and locating the run conf in a fetched tree. (fetch/certoraRun/POU paths are live I/O, +exercised end-to-end, not here.)""" +import json +import tempfile +from pathlib import Path + +from summarization_detector.sources import ( + cut_from_conf, find_run_conf, find_external_call_graph) + + +def test_cut_from_conf_reads_verify_then_parametric(): + with tempfile.TemporaryDirectory() as d: + c = Path(d) / "run.conf" + c.write_text(json.dumps({"verify": "Router:certora/specs/sanity-Router.spec"})) + assert cut_from_conf(c) == "Router" + c.write_text(json.dumps({"parametric_contracts": ["Vault", "Other"]})) # no verify -> parametric + assert cut_from_conf(c) == "Vault" + c.write_text(json.dumps({"files": ["A.sol"]})) # neither -> "" + assert cut_from_conf(c) == "" + + +def test_find_run_conf_prefers_canonical_and_skips_lib(): + with tempfile.TemporaryDirectory() as d: + root = Path(d) + (root / "inputs" / ".certora_sources").mkdir(parents=True) + (root / "inputs" / ".certora_sources" / "run.conf").write_text("{}") + (root / "lib" / "dep").mkdir(parents=True) + (root / "lib" / "dep" / "run.conf").write_text("{}") # must be ignored + found = find_run_conf(root) + assert found == root / "inputs" / ".certora_sources" / "run.conf" + + +def test_find_external_call_graph_optional(): + with tempfile.TemporaryDirectory() as d: + root = Path(d) + assert find_external_call_graph(root) is None # absent -> None (no gate) + rpt = root / "Reports" + rpt.mkdir() + (rpt / "externalCallGraph.json").write_text("{}") + assert find_external_call_graph(root) == rpt / "externalCallGraph.json" + + diff --git a/summarization_detector/tests/test_surviving.py b/summarization_detector/tests/test_surviving.py new file mode 100644 index 00000000..ba0671b3 --- /dev/null +++ b/summarization_detector/tests/test_surviving.py @@ -0,0 +1,95 @@ +"""Signal 4 (surviving hostile primitives) — pure/offline: the generic-vs-curated +classifier, the linear-scaling exclusion, the surviving_hostile aggregation (ghost exclusion + per-primitive +reaching-method roll-up), and the detect_from fusion into an enriched candidate. The live POU fetch +(fetch_surviving_graphs) is exercised end-to-end against real runs, not here.""" +from summarization_detector.detect import ( + DifficultyReport, classify_hostile, detect_from, surviving_hostile, _entry_of_rule, +) + + +def test_classify_generic_categories(): + def cat(name: str) -> str: + m = classify_hostile(name) + assert m is not None + return m.category + assert cat("BitLib.fls(uint256)") == "bitwise-scan" + assert cat("Foo.popCount(uint256)") == "bitwise-scan" + assert cat("ListLib.sortByKey(ListLib.List)") == "in-memory-sort" + assert cat("SomeLib.pow(uint256,uint256)") == "symbolic-exp" + assert cat("FooMath.mulDivDown(uint256,uint256,uint256)") == "nonlinear-mulDiv" + + +def test_generic_match_suggests_no_summary(): + # a generic (non-curated) match reports the category but leaves the summary to the agent + for name in ("ListLib.sortByKey(ListLib.List)", "BitLib.fls(uint256)"): + m = classify_hostile(name) + assert m is not None and m.curated is False and m.candidate_summary == "" + + +def test_curated_overlay_attaches_a_concrete_summary(): + oz = classify_hostile("Math.mulDiv(uint256,uint256,uint256)") + assert oz is not None and oz.category == "nonlinear-mulDiv" + assert oz.curated is True and oz.candidate_summary != "" + + +def test_generic_camelcase_exp_is_caught_without_hardcoding(): + # a camelCase `…Exp` name is matched structurally (no curated entry → no suggested summary) + e = classify_hostile("PriceLib.computeExp(uint256,uint256)") + assert e is not None and e.category == "symbolic-exp" and e.curated is False and e.candidate_summary == "" + + +def test_linear_constant_scaling_excluded(): + # multiply-by-constant conversions are cheap, not hostile + assert classify_hostile("WadRayMath.bpsToWad(uint256)") is None + assert classify_hostile("WadRayMath.toRay(uint256)") is None + assert classify_hostile("Foo.normalizeDecimals(uint256,uint8,uint8)") is None + + +def _graph(method, names): + return {"rule": f"sanity-{method}-Satisfy_x", "phase": "postOptimize", "procedures": [], + "internalFunctions": [{"name": n, "summarizable": True} for n in names]} + + +def test_surviving_hostile_aggregates_reach_and_excludes_ghosts(): + graphs = [ + _graph("borrow(uint256)", ["ListLib.sortByKey(ListLib.List)", "BitLib.fls(uint256)", + "CVL/Ghost Function 'mulDivUpSummary256(a,b,c)'"]), + _graph("withdraw(uint256)", ["ListLib.sortByKey(ListLib.List)"]), # sort reaches a 2nd method + _graph("getX()", ["Foo.plainGetter()"]), # nothing hostile + ] + h = surviving_hostile(graphs) + assert set(h) == {"ListLib.sortByKey(ListLib.List)", "BitLib.fls(uint256)"} # ghost + non-hostile out + assert sorted(h["ListLib.sortByKey(ListLib.List)"]["reaching_methods"]) == [ + "borrow(uint256)", "withdraw(uint256)"] + assert h["BitLib.fls(uint256)"]["reaching_methods"] == ["borrow(uint256)"] + assert h["ListLib.sortByKey(ListLib.List)"]["category"] == "in-memory-sort" + + +def test_detect_from_fuses_surviving_into_enriched_candidate(): + # the surviving name carries a signature; the candidate key is sig-stripped so it merges cleanly. + # A generic match carries no summary; a curated one carries the concrete text. + surv = surviving_hostile([_graph("borrow(uint256)", ["BitLib.fls(uint256)", + "WadRayMath.rayMul(uint256,uint256)"])]) + rep = detect_from([], DifficultyReport(), cut="Vault", surviving=surv) + fls = next(c for c in rep.candidates if c.function == "BitLib.fls") + assert "bitwise-scan" in fls.signals # the hard-op class IS the signal (no separate category) + assert fls.reaching_count == 1 and fls.candidate_summary == "" + ray = next(c for c in rep.candidates if c.function == "WadRayMath.rayMul") + assert "nonlinear-mulDiv" in ray.signals and ray.candidate_summary != "" + + +def test_entry_of_rule(): + assert _entry_of_rule("sanity-setFlag(uint256,bool,address)-Satisfy_x") == "setFlag(uint256,bool,address)" + # the per-method Assertions graph must strip to the same bare method (not leak the raw rule name) + assert _entry_of_rule("sanity-setFlag(uint256,bool,address)-Assertions") == "setFlag(uint256,bool,address)" + + +def test_surviving_dedups_satisfy_and_assertions_graphs(): + from summarization_detector.detect import surviving_hostile + def g(rule): + return {"rule": rule, "phase": "postOptimize", "procedures": [], + "internalFunctions": [{"name": "BitLib.fls(uint256)", "summarizable": True}]} + # a method's Satisfy + Assertions graphs both keep the primitive -> counted ONCE + out = surviving_hostile([g("sanity-borrow(uint256)-Satisfy_sanity_check_failed_x"), + g("sanity-borrow(uint256)-Assertions")]) + assert out["BitLib.fls(uint256)"]["reaching_methods"] == ["borrow(uint256)"] diff --git a/template_manifest.json b/template_manifest.json index ae195eb4..a3570e41 100644 --- a/template_manifest.json +++ b/template_manifest.json @@ -11,6 +11,18 @@ "template_name": "stub_generation_prompt.j2", "ty_sort": "PartialTemplate" }, + "composer.foundry.author:_FoundryJudgeTemplate": { + "module": "composer.foundry.author", + "qualname": "_FoundryJudgeTemplate", + "template_name": "foundry_feedback_prompt.j2", + "ty_sort": "TypedTemplate" + }, + "composer.foundry.author:_FoundryJudgeSystemTemplate": { + "module": "composer.foundry.author", + "qualname": "_FoundryJudgeSystemTemplate", + "template_name": "foundry_property_judge_system_prompt.j2", + "ty_sort": "TypedTemplate" + }, "composer.foundry.author:_FoundryPropertyGenTemplate": { "module": "composer.foundry.author", "qualname": "_FoundryPropertyGenTemplate", @@ -23,6 +35,12 @@ "template_name": "cvl_kb_index.j2", "ty_sort": "TypedTemplate" }, + "composer.pipeline.ecosystem:EVM_CODE_EXPLORER_TEMPLATE": { + "module": "composer.pipeline.ecosystem", + "qualname": "EVM_CODE_EXPLORER_TEMPLATE", + "template_name": "code_explorer/solidity.j2", + "ty_sort": "TypedTemplate" + }, "composer.pipeline.ecosystem:SOLANA_ANALYSIS_SYSTEM_TEMPLATE": { "module": "composer.pipeline.ecosystem", "qualname": "SOLANA_ANALYSIS_SYSTEM_TEMPLATE", @@ -47,6 +65,48 @@ "template_name": "solana/property_prompt.j2", "ty_sort": "TypedTemplate" }, + "composer.pipeline.ecosystem:SOLANA_CODE_EXPLORER_TEMPLATE": { + "module": "composer.pipeline.ecosystem", + "qualname": "SOLANA_CODE_EXPLORER_TEMPLATE", + "template_name": "code_explorer/solana.j2", + "ty_sort": "TypedTemplate" + }, + "composer.pipeline.ecosystem:SOROBAN_ANALYSIS_SYSTEM_TEMPLATE": { + "module": "composer.pipeline.ecosystem", + "qualname": "SOROBAN_ANALYSIS_SYSTEM_TEMPLATE", + "template_name": "soroban/analysis_system.j2", + "ty_sort": "TypedTemplate" + }, + "composer.pipeline.ecosystem:SOROBAN_ANALYSIS_INITIAL_TEMPLATE": { + "module": "composer.pipeline.ecosystem", + "qualname": "SOROBAN_ANALYSIS_INITIAL_TEMPLATE", + "template_name": "soroban/analysis_prompt.j2", + "ty_sort": "TypedTemplate" + }, + "composer.pipeline.ecosystem:SOROBAN_PROPERTY_SYSTEM_TEMPLATE": { + "module": "composer.pipeline.ecosystem", + "qualname": "SOROBAN_PROPERTY_SYSTEM_TEMPLATE", + "template_name": "soroban/property_system.j2", + "ty_sort": "TypedTemplate" + }, + "composer.pipeline.ecosystem:SOROBAN_PROPERTY_INITIAL_TEMPLATE": { + "module": "composer.pipeline.ecosystem", + "qualname": "SOROBAN_PROPERTY_INITIAL_TEMPLATE", + "template_name": "soroban/property_prompt.j2", + "ty_sort": "TypedTemplate" + }, + "composer.pipeline.ecosystem:SOROBAN_CODE_EXPLORER_TEMPLATE": { + "module": "composer.pipeline.ecosystem", + "qualname": "SOROBAN_CODE_EXPLORER_TEMPLATE", + "template_name": "code_explorer/soroban.j2", + "ty_sort": "TypedTemplate" + }, + "composer.rustapp.session:ProtocolTemplate": { + "module": "composer.rustapp.session", + "qualname": "ProtocolTemplate", + "template_name": "authoring_protocol.j2", + "ty_sort": "TypedTemplate" + }, "composer.spec.cvl_research:_ResearchSys": { "module": "composer.spec.cvl_research", "qualname": "_ResearchSys", @@ -89,6 +149,12 @@ "template_name": "property_generation_prompt.j2", "ty_sort": "TypedTemplate" }, + "composer.spec.source.author:_PropertyGenSysTemplate": { + "module": "composer.spec.source.author", + "qualname": "_PropertyGenSysTemplate", + "template_name": "property_generation_system_prompt.j2", + "ty_sort": "TypedTemplate" + }, "composer.spec.source.design_doc_finder:_FINDER_PROMPT": { "module": "composer.spec.source.design_doc_finder", "qualname": "_FINDER_PROMPT", diff --git a/tests/conftest.py b/tests/conftest.py index 52720899..875a7b27 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -3,8 +3,11 @@ """ +import json import os +import re import uuid +from pathlib import Path # certora_autosetup.setup.setup_summaries hard-exits at IMPORT time when # ANTHROPIC_API_KEY is absent, which would crash test collection for any test @@ -14,6 +17,7 @@ from typing import Any, AsyncIterator, Iterator, Callable, Iterable, TYPE_CHECKING, Sequence from contextlib import asynccontextmanager from dataclasses import dataclass, field +from urllib.parse import urlparse import numpy as np import psycopg @@ -45,12 +49,45 @@ except ImportError: _HAS_TESTCONTAINERS = False +def _external_pg_url() -> str | None: + """Admin DSN of an already-running postgres to use INSTEAD of testcontainers. + + Set by the containerized test flow (docs/crucible-demo.md (PR3)) so DB-backed tests + run inside the crucible container against the compose `postgres` service — no + docker-in-docker. Must be a superuser DSN (tests CREATE ROLE/DATABASE).""" + return os.environ.get("COMPOSER_TEST_PG_URL") or None + + needs_postgres = pytest.mark.skipif( - not _HAS_TESTCONTAINERS, - reason="testcontainers[postgres] not installed", + not (_HAS_TESTCONTAINERS or _external_pg_url()), + reason="no postgres: install testcontainers[postgres] or set COMPOSER_TEST_PG_URL", ) +class _ExternalPostgres: + """Minimal ``PostgresContainer`` stand-in for an already-running postgres, so + DB-backed tests run unchanged against the compose `postgres` service. Only the + surface the tests use is implemented.""" + + def __init__(self, url: str) -> None: + p = urlparse(url) + self.username = p.username or "postgres" + self.password = p.password or "" + self._host = p.hostname or "localhost" + self._port = p.port or 5432 + self._db = (p.path or "/postgres").lstrip("/") or "postgres" + + def get_connection_url(self, host: str | None = None, driver: str | None = "psycopg2") -> str: + scheme = "postgresql" if not driver else f"postgresql+{driver}" + return f"{scheme}://{self.username}:{self.password}@{self._host}:{self._port}/{self._db}" + + def get_container_host_ip(self) -> str: + return self._host + + def get_exposed_port(self, port: int) -> int: + return self._port + + @pytest.fixture(autouse=True) def _isolate_run_summary(): """Keep the run-summary context var from leaking between tests.""" @@ -242,8 +279,13 @@ async def langgraph_db() -> AsyncIterator[LanggraphDBSetup | None]: @pytest.fixture(scope="session") def pg_container() -> Iterator["PostgresContainer | None"]: + ext = _external_pg_url() + if ext: + yield _ExternalPostgres(ext) # type: ignore[misc] + return if not _HAS_TESTCONTAINERS: - return None + yield None + return with PostgresContainer("pgvector/pgvector:pg16") as pg: yield pg @@ -298,7 +340,55 @@ async def pg_database(pg_database_opt: PGAsyncPool | None) -> AsyncIterator[PGAs yield pg_database_opt type ProverToolResponse = ProverReport | str -type ProverMock = Callable[[Iterable[ProverToolResponse]], BaseTool] + +#: rule/invariant declarations of a CVL spec — the ground truth the mocked +#: ``declared_rules_list`` derives from the spec text instead of running +#: certoraRun + the typechecker's ``-listRules``. +SPEC_DECL_RE = re.compile(r"^\s*(?:rule|invariant)\s+([A-Za-z_]\w*)", re.MULTILINE) + + +def conf_of_prover_call(folder: Path, args: list[str]) -> dict: + """The conf json a prover entry point was invoked with: ``args[0]``, which + verify_spec passes project-root-relative.""" + conf_path = Path(args[0]) + if not conf_path.is_absolute(): + conf_path = folder / conf_path + return json.loads(conf_path.read_text()) + + +def spec_of_prover_conf(folder: Path, conf: dict) -> str: + """The text of the spec a prover conf verifies (its ``verify`` target).""" + spec_path = Path(conf["verify"].split(":", 1)[1]) + if not spec_path.is_absolute(): + spec_path = folder / spec_path + return spec_path.read_text() + + +@dataclass +class ProverCall: + """One mocked ``run_prover`` invocation: where it ran, its argv, and the conf + verify_spec wrote for it (snapshotted at call time — the file is unlinked when + the run context exits).""" + folder: Path + args: list[str] + conf: dict + + +class ProverMock: + """Binder over the mocked prover seams: call it with the ``run_prover`` response + script to get the verify_spec tool; ``calls`` records every mocked run.""" + + def __init__( + self, + bind: Callable[[Iterable[ProverToolResponse]], BaseTool], + calls: list[ProverCall], + ) -> None: + self._bind = bind + self.calls = calls + + def __call__(self, l: Iterable[ProverToolResponse]) -> BaseTool: + return self._bind(l) + @pytest.fixture def fake_llm(): @@ -312,18 +402,24 @@ def certora_prover( ) -> ProverMock: response_script : list[ProverToolResponse] | None = None response_ptr = 0 + calls: list[ProverCall] = [] + + async def mock_declared_rules(folder: Path, args: list[str]) -> list[str]: + return SPEC_DECL_RE.findall(spec_of_prover_conf(folder, conf_of_prover_call(folder, args))) async def mock_prover( - *args, **kwargs + folder: Path, args: list[str], *rest, **kwargs ) -> ProverToolResponse: assert response_script is not None nonlocal response_ptr assert response_ptr < len(response_script) + calls.append(ProverCall(folder=folder, args=list(args), conf=conf_of_prover_call(folder, args))) to_ret = response_script[response_ptr] response_ptr += 1 return to_ret - + monkeypatch.setattr("composer.spec.source.prover.run_prover", mock_prover) + monkeypatch.setattr("composer.spec.source.prover.declared_rules_list", mock_declared_rules) monkeypatch.setattr("composer.spec.source.prover.get_stream_writer", lambda: ( lambda _: None )) @@ -340,4 +436,80 @@ def bind_tool(l: Iterable[ProverToolResponse]) -> BaseTool: response_script = list(l) return the_tool - return bind_tool + return ProverMock(bind_tool, calls) + + +# --------------------------------------------------------------------------- +# Rust ABI payloads (``composer.rustapp.wire`` / ``.descriptor``) +# +# The seam carries no defaults on the side that deserializes: both halves ship together, so an +# absent field is a drifted mirror, not an older wheel. That is right for the ABI and wrong for a +# fixture — a test spelling all fifteen descriptor fields to exercise one of them buries what it is +# testing. These builders stand in for the wheel that would have sent the rest, so what they +# produce is exactly what a real wheel emits. Scaffolding, not tolerance. +# --------------------------------------------------------------------------- + +def wire_phase(key: str, label: str, order: int, role: str = "grouping") -> dict[str, Any]: + """One ``PhaseSpec``. ``role`` defaults to grouping — a phase declaring no step of its own.""" + return {"key": key, "label": label, "order": order, "role": role} + + +def wire_required_phases() -> list[dict[str, Any]]: + """The four steps the driver runs itself, which every application must claim. A fresh list, so a + caller can splice its own phase in without reaching into another test's fixture.""" + return [ + wire_phase("analysis", "A", 0, "analysis"), + wire_phase("extraction", "E", 1, "extraction"), + wire_phase("formalization", "F", 2, "formalization"), + wire_phase("report", "R", 3, "report"), + ] + + +def wire_descriptor(**overrides: Any) -> dict[str, Any]: + """A complete ``AppDescriptor`` payload, with ``overrides`` applied.""" + return { + "name": "demoprover", + "header_text": "h", + "ecosystem": "evm", + "backend_tag": "prover", + "backend_guidance": "g", + "analysis_key": "k", + "phases": wire_required_phases(), + "args": [], + "rag_db_default": None, + "event_kinds": [], + "artifact_layout": { + "deliverable_dir": "d", "internal_dir": "i", "report_dir": "r", "artifact_dir": "a", + "artifact_prefix": "p", "artifact_extension": "rs", "property_suffix": "s", + }, + "deliverable_mode": {"mode": "per_component"}, + "serialize_toolchain": False, + "confine_by_default": False, + "component_noun": None, + "check_noun": None, + "evidence_kinds": ["build_failure", "check_output", "counterexample", "reasoned"], + **overrides, + } + + +def wire_check(prop: str, name: str, target: str | None = None) -> dict[str, Any]: + """One check (``Check``); ``target`` null means the check is its own validation target.""" + return {"property": prop, "name": name, "target": target} + + +def wire_verdict(outcome: str, **overrides: Any) -> dict[str, Any]: + """One ``Verdict`` — every diagnostic field null unless ``overrides`` says otherwise.""" + return { + "outcome": outcome, "line": None, "duration_seconds": None, + "unit_file": None, "detail": None, **overrides, + } + + +def wire_prompt(instruction: str, system: str | None = None) -> dict[str, Any]: + """One authoring ``Prompt``.""" + return {"instruction": instruction, "system": system} + + +def wire_workspace_prep(**overrides: Any) -> dict[str, Any]: + """A ``WorkspacePrep`` plan that asks for nothing, plus ``overrides``.""" + return {"files": {}, "toolchain_request": {}, **overrides} diff --git a/tests/solidity_ast/__init__.py b/tests/solidity_ast/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/solidity_ast/test_lenient_parsing.py b/tests/solidity_ast/test_lenient_parsing.py new file mode 100644 index 00000000..e809857c --- /dev/null +++ b/tests/solidity_ast/test_lenient_parsing.py @@ -0,0 +1,120 @@ +"""Corpus-discovered leniency cases: shapes real solc emits that the vendored schema +does not account for (see LENIENT_REQUIRED / DELIBERATELY_OPEN in the conformance +test). Each must parse typed — not as UnknownNode, not as a parse failure — and +round-trip without inventing the absent fields. When the producing solc version is +known, VERSION_GATES turns illegitimate absence into a failure instead of a None.""" + +import pytest + +from certora_autosetup.solidity_ast import ( + AstDump, + ContractDefinition, + EventDefinition, + FunctionDefinition, + Return, + SourceUnit, + VariableDeclaration, +) + + +def test_pre_06_contract_without_abstract_parses() -> None: + contract = ContractDefinition.model_validate( + { + "id": 1, "src": "0:10:0", "nodeType": "ContractDefinition", + "name": "C", "baseContracts": [], "contractDependencies": [], + "contractKind": "contract", "fullyImplemented": True, + "linearizedBaseContracts": [1], "nodes": [], "scope": 0, + } + ) + assert contract.abstract is False + assert "abstract" not in contract.model_dump(exclude_unset=True) + + +def test_return_inside_modifier_body_parses() -> None: + ret = Return.model_validate({"id": 2, "src": "0:7:0", "nodeType": "Return"}) + assert ret.functionReturnParameters is None + assert "functionReturnParameters" not in ret.model_dump(exclude_unset=True) + + +_BARE_CONTRACT = { + "id": 1, "src": "0:10:0", "nodeType": "ContractDefinition", + "name": "C", "baseContracts": [], "contractDependencies": [], + "contractKind": "contract", "fullyImplemented": True, + "linearizedBaseContracts": [1], "nodes": [], "scope": 0, +} + + +def _dump_with(node: dict) -> dict: + return {"a.sol": {"a.sol": {"1": { + "id": 0, "src": "0:10:0", "nodeType": "SourceUnit", + "absolutePath": "a.sol", "exportedSymbols": {}, "nodes": [node], + }}}} + + +def test_version_gate_fails_absent_field_at_or_above_gate() -> None: + data = _dump_with(dict(_BARE_CONTRACT)) # no `abstract` (gate: 0.6.0) + with pytest.raises(ValueError, match="version-gate violation.*abstract"): + AstDump.from_dict(data, on_error="raise", solc_version="0.8.30") + + dump = AstDump.from_dict(data, on_error="raw", solc_version="0.8.30") + [(_, source)] = list(dump.iter_sources()) + assert source.raw_kind == "parse_failed" and "abstract" in (source.parse_error or "") + + +def test_version_gate_allows_absence_below_gate() -> None: + data = _dump_with(dict(_BARE_CONTRACT)) + dump = AstDump.from_dict(data, on_error="raise", solc_version="0.5.17") + [(_, source)] = list(dump.iter_sources()) + assert source.is_parsed + + +def test_version_gate_off_without_version() -> None: + dump = AstDump.from_dict(_dump_with(dict(_BARE_CONTRACT)), on_error="raise") + [(_, source)] = list(dump.iter_sources()) + assert source.is_parsed + + +def test_effective_mutability_derives_from_constant() -> None: + def var(constant: bool) -> VariableDeclaration: + return VariableDeclaration.model_validate({ + "id": 5, "src": "0:1:0", "nodeType": "VariableDeclaration", + "name": "x", "constant": constant, "scope": 1, "stateVariable": True, + "storageLocation": "default", "typeDescriptions": {}, "visibility": "internal", + }) + + assert var(True).mutability is None and var(True).effective_mutability == "constant" + assert var(False).effective_mutability == "mutable" + + +def test_effective_kind_derives_from_04_flags() -> None: + def fn(name: str, is_constructor: bool) -> FunctionDefinition: + return FunctionDefinition.model_validate({ + "id": 6, "src": "0:1:0", "nodeType": "FunctionDefinition", + "name": name, "implemented": True, "isConstructor": is_constructor, + "modifiers": [], "scope": 1, "stateMutability": "nonpayable", + "visibility": "public", + "parameters": {"id": 7, "src": "0:0:0", "nodeType": "ParameterList", "parameters": []}, + "returnParameters": {"id": 8, "src": "0:0:0", "nodeType": "ParameterList", "parameters": []}, + }) + + assert fn("f", False).kind is None and fn("f", False).effective_kind == "function" + assert fn("", True).effective_kind == "constructor" + assert fn("", False).effective_kind == "fallback" + assert fn("f", False).virtual is None + + +def test_file_level_event_definition_is_typed() -> None: + unit = SourceUnit.model_validate( + { + "id": 10, "src": "0:50:0", "nodeType": "SourceUnit", + "absolutePath": "a.sol", "exportedSymbols": {}, + "nodes": [{ + "id": 11, "src": "0:20:0", "nodeType": "EventDefinition", + "name": "E", "anonymous": False, + "parameters": {"id": 12, "src": "0:0:0", "nodeType": "ParameterList", + "parameters": []}, + }], + } + ) + [event] = unit.nodes + assert isinstance(event, EventDefinition) diff --git a/tests/solidity_ast/test_schema_conformance.py b/tests/solidity_ast/test_schema_conformance.py new file mode 100644 index 00000000..93f3fc15 --- /dev/null +++ b/tests/solidity_ast/test_schema_conformance.py @@ -0,0 +1,528 @@ +"""Machine-checks every solidity_ast pydantic model against the vendored JSON Schema. + +The vendored OpenZeppelin ``solidity-ast`` schema (``certora_autosetup/solidity_ast/ +schema/schema.json``) is the source of truth for field sets. This test asserts, per +schema definition and property: presence, requiredness, nullability, discriminator +value, enum values, and a one-level structural kind check of the pydantic annotation. + +The schema-side helpers (``load_schema``/``node_definitions``/``classify_prop``) are +pure and importable without the model modules; everything model-side is imported +lazily via ``models()`` so this file stays usable while the model package is being +built. +""" + +import json +from collections import Counter +from dataclasses import dataclass, field as dc_field, replace +from functools import lru_cache +from importlib import resources +from types import NoneType, UnionType +from typing import Annotated, Any, Literal, Union, get_args, get_origin + +import pytest + +# --------------------------------------------------------------------------- +# Schema side (pure: no model imports) +# --------------------------------------------------------------------------- + + +@lru_cache(maxsize=None) +def load_schema() -> dict[str, Any]: + schema_file = resources.files("certora_autosetup.solidity_ast") / "schema" / "schema.json" + return json.loads(schema_file.read_text(encoding="utf-8")) + + +@lru_cache(maxsize=None) +def node_definitions() -> dict[str, dict[str, Any]]: + """Every schema definition that has a ``nodeType`` property, plus the schema + ROOT (the SourceUnit definition lives at the top level, not under definitions). + """ + schema = load_schema() + defs = { + name: definition + for name, definition in schema["definitions"].items() + if "nodeType" in definition.get("properties", {}) + } + defs["SourceUnit"] = {"properties": schema["properties"], "required": schema["required"]} + return defs + + +@dataclass(frozen=True) +class Shape: + """One classified schema property (nulls stripped out of anyOf into ``nullable``).""" + + kind: str # primitive | enum | ref | array | map | object | union | null + nullable: bool = False + py: type | None = None # primitive + values: tuple[Any, ...] = () # enum + ref: str = "" # ref (definition name) + item: "Shape | None" = None # array + members: tuple["Shape", ...] = () # union + + +_PRIMITIVES = {"string": str, "integer": int, "boolean": bool, "number": float} + + +def classify_prop(prop: dict[str, Any]) -> Shape: + """Classify a schema property into a Shape; raises on any shape the schema does + not actually contain (so schema updates that add new shapes fail loudly). + """ + if "$ref" in prop: + return Shape("ref", ref=prop["$ref"].rsplit("/", 1)[-1]) + if "anyOf" in prop: + non_null = [m for m in prop["anyOf"] if m.get("type") != "null"] + nullable = len(non_null) < len(prop["anyOf"]) + if not non_null: + return Shape("null", nullable=True) + if len(non_null) == 1: + inner = classify_prop(non_null[0]) + return replace(inner, nullable=nullable or inner.nullable) + return Shape( + "union", nullable=nullable, members=tuple(classify_prop(m) for m in non_null) + ) + if "enum" in prop: + return Shape("enum", values=tuple(prop["enum"])) + schema_type = prop.get("type") + if schema_type == "null": + return Shape("null", nullable=True) + if schema_type in _PRIMITIVES: + return Shape("primitive", py=_PRIMITIVES[schema_type]) + if schema_type == "array": + return Shape("array", item=classify_prop(prop["items"])) + if schema_type == "object": + additional = prop.get("additionalProperties") + if isinstance(additional, dict): + return Shape("map", item=classify_prop(additional)) + return Shape("object") + raise ValueError(f"unclassifiable schema property: {json.dumps(prop)[:200]}") + + +def classify_all() -> Counter[str]: + """Classify every property of every node definition; raises if any is + unclassifiable. Returns kind counts ('?' suffix marks nullable shapes). + Runnable standalone, before the model modules exist. + """ + counts: Counter[str] = Counter() + for definition in node_definitions().values(): + for prop in definition["properties"].values(): + shape = classify_prop(prop) + counts[shape.kind + ("?" if shape.nullable else "")] += 1 + return counts + + +# --------------------------------------------------------------------------- +# Model side (lazy imports: unions.py wires and rebuilds all node modules) +# --------------------------------------------------------------------------- + + +class ModelInterface: + def __init__(self) -> None: + from pydantic import BaseModel + + from certora_autosetup.solidity_ast import unions, yul + from certora_autosetup.solidity_ast.base import ( + Mutability, + StateMutability, + StorageLocation, + TypeDescriptions, + UnknownNode, + Visibility, + ) + + self.base_model: type = BaseModel + self.registry: dict[str, type] = unions.MODEL_BY_SCHEMA_DEF + self.unknown_node: type = UnknownNode + self.type_descriptions: type = TypeDescriptions + # How each schema helper definition is transcribed on the python side. + self.ref_to_py: dict[str, Any] = { + "SourceLocation": str, + "TypeDescriptions": TypeDescriptions, + "Visibility": Visibility, + "StateMutability": StateMutability, + "Mutability": Mutability, + "StorageLocation": StorageLocation, + "Expression": unions.Expression, + "Statement": unions.Statement, + "TypeName": unions.TypeName, + "YulStatement": yul.YulStatement, + "YulExpression": yul.YulExpression, + "YulLiteral": yul.YulLiteral, + } + # Classes that legitimately appear inside field annotations; any other + # BaseModel subclass found there is an inline-object helper model. + self.known_classes: frozenset[type] = frozenset(self.registry.values()) | { + TypeDescriptions + } + + +@lru_cache(maxsize=None) +def models() -> ModelInterface: + return ModelInterface() + + +# Model fields allowed to have no schema property backing them. +# certoraRun injects certora_contract_name into the dump (AstNode base field); +# the rest are the <= 0.5 dialect: InlineAssembly.operations (assembly source +# text) and the solc-0.4/0.5 FunctionDefinition flags. +# nativeSrc needs no entry because every Yul definition lists it in the schema. +FIELD_ALLOWLIST = frozenset({ + "certora_contract_name", + "operations", + "isConstructor", + "isDeclaredConst", + "payable", + "superFunction", +}) + +# Definitions whose nodeType tag differs from the registry key: solc emits +# nodeType "YulLiteral" for both literal kinds, discriminated by hexValue/value. +TAG_OVERRIDES = {"YulLiteralValue": "YulLiteral", "YulLiteralHexValue": "YulLiteral"} + + +# --------------------------------------------------------------------------- +# Annotation flattening / atom extraction +# --------------------------------------------------------------------------- + + +def _flat_members(ann: Any) -> list[Any]: + """Union members of an annotation, with Annotated wrappers (pydantic Tag / + Discriminator metadata) and PEP 695 TypeAliasType lazily unwrapped. + """ + while True: + if type(ann).__name__ == "TypeAliasType" and hasattr(ann, "__value__"): + ann = ann.__value__ + continue + if get_origin(ann) is Annotated: + ann = get_args(ann)[0] + continue + break + if get_origin(ann) in (Union, UnionType): + members: list[Any] = [] + for arg in get_args(ann): + members.extend(_flat_members(arg)) + return members + return [ann] + + +@dataclass +class Atoms: + """The one-level structural content of an annotation (or of a Shape).""" + + classes: set[type] = dc_field(default_factory=set) # known models / primitives + helpers: set[type] = dc_field(default_factory=set) # inline-object helper models + literals: set[Any] = dc_field(default_factory=set) + lists: list[Any] = dc_field(default_factory=list) # element annotations / Shapes + dicts: int = 0 + objects: int = 0 # expected-side marker for inline-object schemas + has_none: bool = False + other: list[Any] = dc_field(default_factory=list) + + def merge(self, more: "Atoms") -> None: + self.classes |= more.classes + self.helpers |= more.helpers + self.literals |= more.literals + self.lists.extend(more.lists) + self.dicts += more.dicts + self.objects += more.objects + self.has_none = self.has_none or more.has_none + self.other.extend(more.other) + + +def atoms_of(ann: Any, m: ModelInterface) -> Atoms: + """Flatten a python annotation into Atoms. The deliberate extra UnknownNode + union member is dropped (it is not part of the schema contract). + """ + atoms = Atoms() + for member in _flat_members(ann): + origin = get_origin(member) + if member is NoneType: + atoms.has_none = True + elif origin is Literal: + atoms.literals |= set(get_args(member)) + elif origin is list: + args = get_args(member) + atoms.lists.append(args[0] if args else Any) + elif origin is dict: + atoms.dicts += 1 + elif isinstance(member, type): + if member is m.unknown_node: + pass + elif issubclass(member, m.base_model) and member not in m.known_classes: + atoms.helpers.add(member) + else: + atoms.classes.add(member) + else: + atoms.other.append(member) + return atoms + + +def expected_atoms(shape: Shape, m: ModelInterface) -> Atoms: + """Atoms the schema Shape demands of the annotation (nullability excluded -- + it is checked against requiredness separately). + """ + atoms = Atoms() + if shape.kind == "primitive": + assert shape.py is not None + atoms.classes.add(shape.py) + elif shape.kind == "enum": + atoms.literals |= set(shape.values) + elif shape.kind == "ref": + target = m.ref_to_py.get(shape.ref) + if target is not None: + resolved = atoms_of(target, m) # alias -> literals / union members / class + resolved.has_none = False # alias-internal nullability is not the field's + atoms.merge(resolved) + elif shape.ref in m.registry: + atoms.classes.add(m.registry[shape.ref]) + else: + atoms.other.append(f"unmapped $ref {shape.ref}") + elif shape.kind == "union": + for member in shape.members: + atoms.merge(expected_atoms(member, m)) + elif shape.kind == "array": + atoms.lists.append(shape.item) + elif shape.kind == "map": + atoms.dicts += 1 + elif shape.kind == "object": + atoms.objects += 1 + elif shape.kind == "null": + pass + return atoms + + +def _compare_atoms(actual: Atoms, expected: Atoms, where: str) -> list[str]: + """One-level comparison; array elements are compared one level deeper, maps and + inline objects are not recursed into. + """ + errors: list[str] = [] + if actual.classes != expected.classes: + errors.append( + f"{where}: annotation classes {sorted(c.__name__ for c in actual.classes)} " + f"!= schema {sorted(c.__name__ for c in expected.classes)}" + ) + if actual.literals != expected.literals: + errors.append( + f"{where}: Literal values {sorted(map(str, actual.literals))} " + f"!= schema enum {sorted(map(str, expected.literals))}" + ) + if expected.objects and not (actual.helpers or actual.dicts): + errors.append(f"{where}: schema inline object needs a helper BaseModel or dict") + if not expected.objects and actual.helpers: + errors.append( + f"{where}: unexpected helper model(s) " + f"{sorted(h.__name__ for h in actual.helpers)}" + ) + if expected.dicts and not actual.dicts: + errors.append(f"{where}: schema map needs a dict[...] annotation") + if actual.dicts and not (expected.dicts or expected.objects): + errors.append(f"{where}: unexpected dict annotation") + if expected.other: + errors.append(f"{where}: {expected.other}") + if actual.other: + errors.append(f"{where}: unrecognized annotation member(s) {actual.other}") + return errors + + +def check_shape(shape: Shape, ann: Any, m: ModelInterface, where: str) -> list[str]: + actual = atoms_of(ann, m) + expected = expected_atoms(shape, m) + errors = _compare_atoms(actual, expected, where) + + if expected.lists: + if not actual.lists: + errors.append(f"{where}: schema array needs a list[...] annotation") + else: + # Compare all list elements jointly, so both list[A] | list[B] and + # list[A | B] transcriptions of an anyOf-of-arrays are accepted. + elem_expected = Atoms() + elem_nullable = False + for item_shape in expected.lists: + assert isinstance(item_shape, Shape) + elem_expected.merge(expected_atoms(item_shape, m)) + elem_nullable = elem_nullable or item_shape.nullable + elem_actual = Atoms() + for elem_ann in actual.lists: + elem_actual.merge(atoms_of(elem_ann, m)) + errors += _compare_atoms(elem_actual, elem_expected, where + " (element)") + if elem_nullable and not elem_actual.has_none: + errors.append(f"{where}: array items are nullable, element lacks | None") + if elem_actual.has_none and not elem_nullable: + errors.append(f"{where}: element allows None but schema items are not nullable") + elif actual.lists: + errors.append(f"{where}: unexpected list annotation") + return errors + + +# --------------------------------------------------------------------------- +# Per-property check +# --------------------------------------------------------------------------- + + +def field_for(model: type, prop: str) -> Any: + for field_name, info in model.model_fields.items(): # type: ignore[attr-defined] + if field_name == prop or info.alias == prop: + return info + return None + + +# Fields deliberately WIDER than the schema, all validated against real dumps +# (fixtures for solc 0.4.26 / 0.5.17 and the project corpus sweep). Presence and +# requiredness are still checked; only the shape check is waived: +# - InlineAssembly.evmVersion/flags: version-freshness enums — a closed transcription +# would demote every assembly-containing source on the first solc release the +# vendored schema lags behind. +# - SourceUnit.nodes: the schema's union lacks EventDefinition, but solc >= 0.8.22 +# allows file-level events. +# - documentation on Contract/Function/Modifier/EventDefinition: plain NatSpec string +# in dumps from solc <= 0.5 (the StructuredDocumentation node form is 0.6+). +# - ElementaryTypeNameExpression.typeName: plain string in dumps from solc <= 0.5. +# - InlineAssembly.externalReferences: solc <= 0.5 items are keyed by identifier name. +DELIBERATELY_OPEN = { + ("InlineAssembly", "evmVersion"), + ("InlineAssembly", "flags"), + ("SourceUnit", "nodes"), + ("ContractDefinition", "documentation"), + ("FunctionDefinition", "documentation"), + ("ModifierDefinition", "documentation"), + ("EventDefinition", "documentation"), + ("ElementaryTypeNameExpression", "typeName"), + ("InlineAssembly", "externalReferences"), +} + +# Schema-required fields the models default instead: solc omits them in situations +# the schema does not account for — either below the 0.6 floor (lenient-older +# policy: typed parsing must still work) or in corners the schema over-requires. +# Shape is still checked. exclude_unset round-trips keep the absence loyal. +# - ContractDefinition.abstract, {Function,Modifier}Definition.virtual, +# FunctionCall.tryCall, VariableDeclaration.mutability: concepts added in 0.6.x. +# - FunctionDefinition.kind: added in 0.5 (0.4 uses isConstructor). +# - InlineAssembly.AST/evmVersion: absent in the <= 0.5 assembly dialect. +# - Return.functionReturnParameters: omitted for `return;` inside a modifier body. +# - MemberAccess.isLValue: omitted by solc 0.7.2 (only) on enum-member accesses — +# a bug window, present both before and after, so it cannot be a version gate. +LENIENT_REQUIRED = { + ("MemberAccess", "isLValue"), + ("ContractDefinition", "abstract"), + ("FunctionDefinition", "virtual"), + ("FunctionDefinition", "kind"), + ("ModifierDefinition", "virtual"), + ("FunctionCall", "tryCall"), + ("VariableDeclaration", "mutability"), + ("InlineAssembly", "AST"), + ("InlineAssembly", "evmVersion"), + ("Return", "functionReturnParameters"), +} + + +def check_property( + model: type, prop: str, spec: dict[str, Any], required: bool, m: ModelInterface +) -> list[str]: + where = f"{model.__name__}.{prop}" + info = field_for(model, prop) + if info is None: + return [f"{where}: field missing (no field named or aliased '{prop}')"] + + errors: list[str] = [] + shape = classify_prop(spec) + nullable = shape.nullable or shape.kind == "null" + has_none = atoms_of(info.annotation, m).has_none + + if required: + if (model.__name__, prop) in LENIENT_REQUIRED: + if info.is_required(): + errors.append(f"{where}: listed in LENIENT_REQUIRED but has no default") + else: + if not info.is_required(): + errors.append(f"{where}: schema-required but field has a default") + if nullable and not has_none: + errors.append(f"{where}: required-but-nullable, annotation lacks | None") + if not nullable and has_none: + errors.append(f"{where}: required non-nullable, annotation must not allow None") + else: + if info.is_required(): + errors.append(f"{where}: schema-optional but field is required") + elif info.default is not None: + errors.append(f"{where}: schema-optional, default must be None (got {info.default!r})") + if not has_none: + errors.append(f"{where}: schema-optional, annotation lacks | None") + + if prop == "nodeType": + if get_args(info.annotation) != (shape.values[0],): + errors.append( + f"{where}: get_args(annotation) == {get_args(info.annotation)!r}, " + f"expected ({shape.values[0]!r},)" + ) + elif (model.__name__, prop) in DELIBERATELY_OPEN: + pass + elif shape.kind != "null": # a pure-null property only constrains nullability + errors += check_shape(shape, info.annotation, m, where) + return errors + + +# --------------------------------------------------------------------------- +# Tests +# --------------------------------------------------------------------------- + +DEF_NAMES = sorted(node_definitions()) + + +def test_classifier_covers_every_schema_shape() -> None: + """Pure schema test (no models): every property shape is classifiable.""" + counts = classify_all() + assert sum(counts.values()) == sum( + len(d["properties"]) for d in node_definitions().values() + ) + + +def test_registry_coverage_both_directions() -> None: + m = models() + schema_names = set(node_definitions()) + model_names = set(m.registry) + missing = schema_names - model_names + extra = model_names - schema_names + assert not missing and not extra, ( + f"MODEL_BY_SCHEMA_DEF mismatch: missing={sorted(missing)} extra={sorted(extra)}" + ) + + +@pytest.mark.parametrize("def_name", DEF_NAMES) +def test_definition_conforms_to_schema(def_name: str) -> None: + m = models() + model = m.registry.get(def_name) + assert model is not None, f"no model registered for schema definition {def_name}" + definition = node_definitions()[def_name] + required = set(definition.get("required", [])) + errors: list[str] = [] + for prop, spec in definition["properties"].items(): + errors += check_property(model, prop, spec, prop in required, m) + assert not errors, "\n".join(errors) + + +@pytest.mark.parametrize("def_name", DEF_NAMES) +def test_no_fields_beyond_schema(def_name: str) -> None: + m = models() + model = m.registry.get(def_name) + assert model is not None, f"no model registered for schema definition {def_name}" + props = set(node_definitions()[def_name]["properties"]) + stray = [ + info.alias or field_name + for field_name, info in model.model_fields.items() + if (info.alias or field_name) not in props + and (info.alias or field_name) not in FIELD_ALLOWLIST + ] + assert not stray, f"{model.__name__}: fields with no schema property: {sorted(stray)}" + + +def test_node_type_tags_match_registry_keys() -> None: + m = models() + errors: list[str] = [] + for def_name, model in m.registry.items(): + expected_tag = TAG_OVERRIDES.get(def_name, def_name) + info = model.model_fields.get("nodeType") + if info is None: + errors.append(f"{def_name}: model {model.__name__} has no nodeType field") + continue + tags = get_args(info.annotation) + if tags != (expected_tag,): + errors.append(f"{def_name}: nodeType Literal {tags!r} != ({expected_tag!r},)") + assert not errors, "\n".join(errors) diff --git a/tests/solidity_ast/test_unions.py b/tests/solidity_ast/test_unions.py new file mode 100644 index 00000000..bd127e32 --- /dev/null +++ b/tests/solidity_ast/test_unions.py @@ -0,0 +1,47 @@ +"""Drift guards for the hand-maintained union wiring in unions.py.""" + +from typing import get_args + +from pydantic import BaseModel + +from certora_autosetup.solidity_ast import unions +from certora_autosetup.solidity_ast.base import UNKNOWN_TAG, UnknownNode + +UNION_TO_TAGSET = { + "Expression": unions._EXPRESSION_TAGS, + "Statement": unions._STATEMENT_TAGS, + "TypeName": unions._TYPENAME_TAGS, + "SourceUnitNode": unions._SOURCEUNITNODE_TAGS, + "ContractBodyNode": unions._CONTRACTBODYNODE_TAGS, + "Node": unions._NODE_TAGS, +} + + +def _tags_of(alias: object) -> set[str]: + """The Tag names attached to a union alias's members (excluding the fallback).""" + union_type, _discriminator = get_args(alias) + tags = set() + for member in get_args(union_type): + _member_type, tag = get_args(member) + tags.add(tag.tag) + return tags - {UNKNOWN_TAG} + + +def test_union_tag_sets_match_members() -> None: + """A member whose tag is missing from the discriminator's frozenset would be + silently routed to UnknownNode — assert the hand-written sets cannot drift.""" + for name, tagset in UNION_TO_TAGSET.items(): + alias = getattr(unions, name) + assert _tags_of(alias) == set(tagset), name + + +def test_every_union_has_unknown_fallback() -> None: + for name in UNION_TO_TAGSET: + union_type, _ = get_args(getattr(unions, name)) + members = {get_args(m)[0] for m in get_args(union_type)} + assert UnknownNode in members, name + + +def test_registry_classes_are_models() -> None: + for def_name, cls in unions.MODEL_BY_SCHEMA_DEF.items(): + assert isinstance(cls, type) and issubclass(cls, BaseModel), def_name diff --git a/tests/test_autoprove_integration.py b/tests/test_autoprove_integration.py index 6493ffaf..e333f4e2 100644 --- a/tests/test_autoprove_integration.py +++ b/tests/test_autoprove_integration.py @@ -15,7 +15,7 @@ import json from pathlib import Path from types import SimpleNamespace -from typing import cast +from typing import Callable, cast import pytest @@ -47,6 +47,8 @@ "solc": "solc", "verify": "Counter:certora/specs/sanity-Counter.spec", "wait_for_results": "none", + "server": "production", + "prover_version": "master" } # AutoSetup's summaries spec, relative to certora/ (the SetupSuccess contract). _SUMMARIES_REL = "specs/summaries/Counter_base_summaries.spec" @@ -83,13 +85,17 @@ def _make_args(rag_conn: str, scenario_dir: Path, system_doc: str | None) -> Aut main_contract=f"{scenario_dir / "src/Counter.sol"}:Counter", system_doc=system_doc, max_concurrent=4, + max_cpu_tasks=2, cache_ns=None, memory_ns=None, cloud=True, interactive=False, threat_model=None, + extra_context=None, recursion_limit=100, max_bug_rounds=1, + # Part of the AutoProveArgs surface (`--budget`); these runs are unbudgeted. + budget=None, rag_db=rag_conn, # Model-config fields: only read through ``get_provider_for(tiered=args)``, # which the tape patches to ignore them, so the values are inert — present @@ -100,15 +106,22 @@ def _make_args(rag_conn: str, scenario_dir: Path, system_doc: str | None) -> Aut thinking_tokens=2048, memory_tool=False, interleaved_thinking=False, + time_budget=None )) -def _install_mocks(monkeypatch, scenario_dir: Path) -> None: +def _install_mocks( + monkeypatch, + scenario_dir: Path, + tape_installer: Callable[[], object] = lambda: install_harness_tape(with_delay=False), +) -> None: """LLM / AutoSetup / embedding mocks (undone per test by ``monkeypatch``). The databases themselves — and the host/port connection redirection — are handled once - per session by the ``langgraph_db`` fixture.""" - # Mock only the LLM (Counter tape) + disable the agent-index cache. - install_harness_tape(with_delay=False) + per session by the ``langgraph_db`` fixture. ``tape_installer`` selects which + tape backs the fake LLM (default: the main Counter tape); the sibling + integration tests pass their variant installers through it.""" + # Mock only the LLM (the selected tape) + disable the agent-index cache. + tape_installer() # pipeline.cli imported `get_provider_for` by name, so install_harness_tape's # patch of registry.get_provider_for doesn't reach that binding — rebind it here. import composer.llm.registry as registry @@ -186,10 +199,13 @@ class _Crasher: properties_key="bar" ) - def __init__(self, store, _ignored): + def __init__(self, store, _opts_ignored, _editing_ignored, _analysis_ignored): self.artiface_store = store + async def preflight(self, *args, **kwargs): + return None + async def prepare_system( self, *args, **kwargs ): diff --git a/tests/test_autoprove_report.py b/tests/test_autoprove_report.py index 6c3507ec..542a9d49 100644 --- a/tests/test_autoprove_report.py +++ b/tests/test_autoprove_report.py @@ -21,9 +21,10 @@ from langchain_core.runnables import Runnable, RunnableLambda from composer.spec.types import PropertyFormulation, PropertyType -from composer.spec.cvl_generation import GeneratedCVL, PropertyRuleMapping, SkippedProperty +from composer.authoring.state import SkippedProperty +from composer.spec.cvl_generation import GeneratedCVL, PropertyRuleMapping -from composer.pipeline.core import Delivered +from composer.pipeline.core import Curtailed, Delivered from composer.spec.source.artifacts import ProverArtifactStore from composer.spec.source.report import build @@ -35,7 +36,7 @@ ) from composer.spec.source.report.render import render_html from composer.spec.source.report.schema import ( - AutoProverReport, CoverageReport, Finding, FindingProvenance, FormalizedProperty, + AutoProverReport, CoverageReport, CurtailedComponent, CurtailedSkip, DraftedProperty, Finding, FindingProvenance, FormalizedProperty, GaveUpComponent, GroupStatus, ImpactLevel, IssueContent, LikelihoodLevel, Outcome, PropertyGroup, RuleVerdict, SeverityTier, SkippedClaim, ) @@ -105,6 +106,21 @@ def _input( formalized=Delivered(result, pathlib.Path(unit_file)) if result is not None else None) +def _curtailed_input( + name, props, result: GeneratedCVL | None, link: str | None = None, detail: str | None = None, +) -> ReportComponentInput[GeneratedCVL]: + """A budget-curtailed component: ``result`` is the partial the author published under lifted + gates (quarantined on disk), or ``None`` when the run stopped before anything was published.""" + partial = ( + Delivered( + deliverable=pathlib.Path(f"autospec_{name}.spec.unverified"), + result=result.model_copy(update={"final_link": link}), + ) + if result is not None else None + ) + return ReportComponentInput(name=name, props=props, formalized=Curtailed(partial, detail)) + + def _fp(component, title, refs, desc="d", sort: PropertyType = "safety_property") -> FormalizedProperty: return FormalizedProperty(component=component, title=title, sort=sort, description=desc, rule_refs=refs) @@ -149,7 +165,7 @@ async def test_collect_joins_properties_to_rules_and_verdicts(): _fake_check("countEqualsSum", NodeStatus.VIOLATED, line=40), ]}) - properties, rules, skipped, gave_up, dropped = await collect( + properties, rules, skipped, gave_up, curtailed, dropped = await collect( [_input("Increment", "autospec_Increment.spec", props, gen)], fetch_verdicts=fetch) assert [p.title for p in properties] == ["count_increases", "count_eq_sum"] @@ -160,7 +176,7 @@ async def test_collect_joins_properties_to_rules_and_verdicts(): assert r.outcome == Outcome.GOOD and r.line == 12 and r.duration_seconds == 1.5 assert r.prover_link == "L1" assert by_ref[("autospec_Increment.spec", "countEqualsSum")].outcome == Outcome.BAD - assert skipped == [] and gave_up == [] and dropped == 0 + assert skipped == [] and gave_up == [] and curtailed == [] and dropped == 0 @pytest.mark.asyncio @@ -169,7 +185,7 @@ async def test_collect_splits_skipped_property_into_gap(): gen = _gen({"p_done": ["r1"]}, skipped={"p_skip": "needs a ghost"}) fetch = _fetcher({"L1": [_fake_check("r1", NodeStatus.VERIFIED)]}) - properties, _rules, skipped, gave_up, _dropped = await collect( + properties, _rules, skipped, gave_up, _curtailed, _dropped = await collect( [_input("C", "autospec_C.spec", props, gen)], fetch_verdicts=fetch) assert [p.title for p in properties] == ["p_done"] @@ -182,13 +198,69 @@ async def test_collect_none_result_is_a_gap(): """A component with no result (the caller maps both give-up and crash to ``None``) is a formalization gap — all its properties unimplemented, no per-property reason.""" props = [_prop("p1", "d1")] - properties, rules, skipped, gave_up, dropped = await collect( + properties, rules, skipped, gave_up, curtailed, dropped = await collect( [_input("C", "autospec_C.spec", props, None)], fetch_verdicts=_fetcher({})) - assert properties == [] and rules == [] and skipped == [] and dropped == 0 + assert properties == [] and rules == [] and skipped == [] and curtailed == [] and dropped == 0 assert [g.component for g in gave_up] == ["C"] assert [p.title for p in gave_up[0].properties] == ["p1"] +@pytest.mark.asyncio +async def test_collect_routes_curtailed_component_without_fetching(): + """A budget-curtailed component contributes nothing to properties/rules; its properties are + partitioned by disposition into the appendix record, and its verdicts are never fetched (the + fetcher spy would record the call — and its link deliberately maps to a verdict that would + otherwise register a rule).""" + good = _input("C", "autospec_C.spec", [_prop("p1", "d1")], _gen({"p1": ["r1"]}, link="Lc")) + cut = _curtailed_input( + "D", + [_prop("q_drafted", "dq"), _prop("q_skip", "ds"), _prop("q_lost", "dl")], + _gen({"q_drafted": ["draft_rule"]}, skipped={"q_skip": "budget exhausted"}), + link="Ld", + ) + base = _fetcher({ + "Lc": [_fake_check("r1", NodeStatus.VERIFIED)], + "Ld": [_fake_check("draft_rule", NodeStatus.VERIFIED, file="autospec_D.spec.unverified")], + }) + fetched: list[str] = [] + + async def fetch(formalized): + fetched.append(formalized.unit_file) + return await base(formalized) + + properties, rules, skipped, gave_up, curtailed, dropped = await collect( + [good, cut], fetch_verdicts=fetch) + + assert fetched == ["autospec_C.spec"] + assert [p.title for p in properties] == ["p1"] + assert [r.name for r in rules] == ["r1"] + assert skipped == [] and gave_up == [] and dropped == 0 + (c,) = curtailed + assert c.component == "D" + assert c.artifact == "autospec_D.spec.unverified" + assert c.run_link == "Ld" + assert [(d.title, d.units) for d in c.drafted] == [("q_drafted", ["draft_rule"])] + assert [(s.title, s.reason) for s in c.skipped] == [("q_skip", "budget exhausted")] + assert [p.title for p in c.unattempted] == ["q_lost"] + + +@pytest.mark.asyncio +async def test_collect_curtailed_without_partial_is_all_unattempted(): + """A hard budget stop published nothing: no artifact, no run link, every inferred property + lands in ``unattempted``.""" + props = [_prop("p1", "d1"), _prop("p2", "d2")] + properties, rules, _s, gave_up, curtailed, _d = await collect( + [_curtailed_input("C", props, None, detail="Token cost budget exhausted")], + fetch_verdicts=_fetcher({}), + ) + assert properties == [] and rules == [] and gave_up == [] + (c,) = curtailed + assert c.artifact is None and c.run_link is None + assert c.detail == "Token cost budget exhausted" + assert c.drafted == [] and c.skipped == [] + assert [p.title for p in c.unattempted] == ["p1", "p2"] + + @pytest.mark.asyncio async def test_collect_drops_and_counts_orphan_rules(): """A rule the prover reported but no property maps to is dropped and counted.""" @@ -197,7 +269,7 @@ async def test_collect_drops_and_counts_orphan_rules(): _fake_check("r1", NodeStatus.VERIFIED), _fake_check("sanity_helper", NodeStatus.VERIFIED), # referenced by nothing ]}) - _props, rules, _skipped, _gave_up, dropped = await collect( + _props, rules, _skipped, _gave_up, _curtailed, dropped = await collect( [_input("C", "autospec_C.spec", [_prop("p1", "d1")], gen)], fetch_verdicts=fetch) assert [r.name for r in rules] == ["r1"] assert dropped == 1 @@ -207,7 +279,7 @@ async def test_collect_drops_and_counts_orphan_rules(): async def test_collect_backfills_unknown_for_unproven_referenced_rule(): gen = _gen({"p1": ["r1"]}) fetch = _fetcher({"L1": []}) # prover reported no checks - properties, rules, _s, _g, dropped = await collect( + properties, rules, _s, _g, _c, dropped = await collect( [_input("C", "autospec_C.spec", [_prop("p1", "d1")], gen)], fetch_verdicts=fetch) assert [(r.name, r.outcome, r.spec_file) for r in rules] == [("r1", Outcome.UNKNOWN, "autospec_C.spec")] assert properties[0].rule_refs == [("autospec_C.spec", "r1")] @@ -313,7 +385,7 @@ def test_validate_property_in_two_groups_raises(): groups = [_pg("g1", [("C", "p1")]), _pg("g2", [("C", "p1")])] with pytest.raises(ValidationError, match="multiple groups"): validate(properties=props, rules=[_rv("s.spec", "a")], groups=groups, - skipped=[], gave_up=[], dropped_orphan_rules=0) + skipped=[], gave_up=[], curtailed=[], dropped_orphan_rules=0) def test_validate_unknown_property_member_raises(): @@ -321,14 +393,14 @@ def test_validate_unknown_property_member_raises(): groups = [_pg("g", [("C", "ghost")])] with pytest.raises(ValidationError, match="don't exist"): validate(properties=props, rules=[_rv("s.spec", "a")], groups=groups, - skipped=[], gave_up=[], dropped_orphan_rules=0) + skipped=[], gave_up=[], curtailed=[], dropped_orphan_rules=0) def test_validate_property_in_no_group_is_soft(): props = [_fp("C", "p1", [("s.spec", "a")]), _fp("C", "p2", [("s.spec", "b")])] groups = [_pg("g", [("C", "p1")])] cov = validate(properties=props, rules=[_rv("s.spec", "a"), _rv("s.spec", "b")], - groups=groups, skipped=[], gave_up=[], dropped_orphan_rules=0) + groups=groups, skipped=[], gave_up=[], curtailed=[], dropped_orphan_rules=0) assert cov.property_coverage_complete is False assert cov.properties_in_no_group == [("C", "p2")] @@ -340,7 +412,7 @@ def test_validate_reports_rules_spanning_groups_as_stat(): p2 = _fp("C", "p2", [("s.spec", "shared")]) groups = [_pg("g1", [("C", "p1")]), _pg("g2", [("C", "p2")])] cov = validate(properties=[p1, p2], rules=[_rv("s.spec", "shared")], groups=groups, - skipped=[], gave_up=[], dropped_orphan_rules=2) + skipped=[], gave_up=[], curtailed=[], dropped_orphan_rules=2) assert cov.rules_spanning_multiple_groups == ["shared"] assert cov.dropped_orphan_rules == 2 @@ -350,9 +422,12 @@ def test_validate_carries_gap_counts(): sk = [SkippedClaim(component="C", title="s1", sort="safety_property", description="d", reason="r")] gu = [GaveUpComponent(component="D", properties=[_prop("x", "d")])] + cu = [CurtailedComponent(component="E", unattempted=[_prop("y", "d")])] cov = validate(properties=[p1], rules=[_rv("s.spec", "a")], groups=[_pg("g", [("C", "p1")])], - skipped=sk, gave_up=gu, dropped_orphan_rules=3) + skipped=sk, gave_up=gu, curtailed=cu, dropped_orphan_rules=3) assert (cov.skipped_count, cov.gave_up_component_count, cov.dropped_orphan_rules) == (1, 1, 3) + assert cov.curtailed_component_count == 1 + assert any("cut short by the run budget" in w for w in cov.warnings) assert cov.property_coverage_complete is True @@ -381,6 +456,24 @@ def _mini_report() -> AutoProverReport: skipped=skipped, coverage=cov) +def test_render_html_shows_finding_message(): + # A backend diagnostic (a counterexample / failed-assertion message, for a backend that + # supplies one) is surfaced on the rule row, so a non-GOOD verdict explains itself. + p1 = _fp("C", "p_dep", [("c.spec", "c_deposit")], desc="deposit increases balance") + rules = [RuleVerdict(name="c_deposit", spec_file="c.spec", outcome=Outcome.BAD, + message="crash abc: deposit(5) - expected 105 got 100")] + groups = [PropertyGroup(slug="deposits", title="Deposits", description="d", + status=GroupStatus.BAD, members=[("C", "p_dep")])] + cov = CoverageReport(total_properties=1, total_rules=1, total_groups=1, + properties_per_group_min=1, properties_per_group_max=1, + property_coverage_complete=True) + h = render_html(AutoProverReport(contract_name="Vault", backend="foundry", + properties=[p1], rules=rules, groups=groups, + skipped=[], coverage=cov)) + assert 'class="finding"' in h + assert "crash abc: deposit(5) - expected 105 got 100" in h + + def test_render_html_group_rows_and_edge_labels(): h = render_html(_mini_report()) assert "deposit-openness" in h and "Deposit is open" in h @@ -436,6 +529,36 @@ def test_render_html_omits_link_column_without_links(): assert "Prover runs" not in h +def test_render_html_budget_appendix(): + """A curtailed component renders as the budget appendix: status chip, counts-first summary, + quarantined artifact path, per-property dispositions, and the footer count.""" + base = _mini_report() + report = base.model_copy(update={ + "curtailed_components": [CurtailedComponent( + component="D", + artifact="autospec_D.spec.unverified", + drafted=[DraftedProperty(title="q1", sort="safety_property", + description="drafted claim", units=["rq"])], + skipped=[CurtailedSkip(title="q2", sort="safety_property", + description="skipped claim", reason="budget exhausted")], + unattempted=[_prop("q3", "never reached")], + )], + "coverage": base.coverage.model_copy(update={"curtailed_component_count": 1}), + }) + h = render_html(report) + assert "Appendix: cut short by the run budget" in h + assert "partial draft published" in h + assert "autospec_D.spec.unverified" in h + assert "Of 3 inferred properties: 1 drafted but never verified, 1 skipped, 1 never attempted." in h + assert "Drafted — unverified" in h and "Not attempted" in h + assert "budget exhausted" in h and "never reached" in h + assert "1 component(s) were cut short by the run budget" in h # footer + + +def test_render_html_without_curtailed_has_no_appendix(): + assert "Appendix: cut short" not in render_html(_mini_report()) + + # --------------------------------------------------------------------------- # build orchestrator (async) # --------------------------------------------------------------------------- @@ -511,6 +634,59 @@ async def test_build_surfaces_skipped_and_gave_up_gaps(tmp_path): assert report.coverage.skipped_count == 1 and report.coverage.gave_up_component_count == 1 +@pytest.mark.asyncio +async def test_build_curtailed_is_appendixed_not_grouped(tmp_path): + """A curtailed component stays out of properties/groups/prover_links and lands in the + appendix + coverage count/warning; the delivered component grounds the report as usual.""" + gen = _gen({"p1": ["r1"]}) + fetch = _fetcher({"L1": [_fake_check("r1", NodeStatus.VERIFIED)]}) + llm = _StructuredStubModel(output=GroupingResult(groups=[PropertyGroupDraft( + slug="g", title="G", description="d", members=[("C", "p1")])])) + + report = await build.build_report( + contract_name="C", + backend="prover", + components=[ + _input("C", "autospec_C.spec", [_prop("p1", "d1")], gen), + _curtailed_input("D", [_prop("q", "d")], _gen({"q": ["rq"]}), link="Lq"), + ], + llm=llm, fetch_verdicts=fetch, + ) + + assert {p.key for p in report.properties} == {("C", "p1")} + assert all(("D", "q") not in g.members for g in report.groups) + (c,) = report.curtailed_components + assert c.component == "D" and [d.title for d in c.drafted] == ["q"] + assert report.coverage.curtailed_component_count == 1 + assert any("cut short by the run budget" in w for w in report.coverage.warnings) + # The curtailed partial's stale run link stays out of the header map. + assert "D" not in report.prover_links + + +@pytest.mark.asyncio +async def test_build_all_curtailed_skips_the_grouping_call(monkeypatch): + """With zero formalized properties there is nothing to group: the grouping LLM must not be + consulted (the exploding stub + re-raise mode make a stray call fail loudly), and the report + degrades to no groups + a populated appendix.""" + monkeypatch.setattr(build, "RERAISE_REPORT_FAILURES", True) + + class _ExplodingModel(_StructuredStubModel): + def with_structured_output(self, schema, **kwargs): # type: ignore[override] + raise AssertionError("grouping LLM consulted despite no formalized properties") + + report = await build.build_report( + contract_name="C", + backend="prover", + components=[_curtailed_input("C", [_prop("p1", "d1")], None, detail="budget exhausted")], + llm=_ExplodingModel(output=GroupingResult(groups=[])), + fetch_verdicts=_fetcher({}), + ) + + assert report.properties == [] and report.groups == [] and report.rules == [] + (c,) = report.curtailed_components + assert c.artifact is None and [p.title for p in c.unattempted] == ["p1"] + assert report.coverage.curtailed_component_count == 1 + assert report.prover_links == {} # --------------------------------------------------------------------------- # findings (violated rules -> audit-issue findings) # --------------------------------------------------------------------------- diff --git a/tests/test_build_system_choice.py b/tests/test_build_system_choice.py new file mode 100644 index 00000000..464d5c2b --- /dev/null +++ b/tests/test_build_system_choice.py @@ -0,0 +1,255 @@ +"""Choosing which build system's artifacts to read. + +Two decisions, both made from what is on disk rather than from config files alone: +``BuildSystemDetector`` picking between a Foundry and a Hardhat config that sit side by +side, and ``ContractExtractor`` picking the directory those artifacts live in when the +project configures its output somewhere other than the default. +""" + +import json +from pathlib import Path + +import pytest + +from certora_autosetup.parsers.build_system_detector import BuildSystem, BuildSystemDetector +from certora_autosetup.parsers.foundry import FoundryContractExtractor + + +def _foundry_config(project: Path, out: str | None = None) -> None: + body = "[profile.default]\n" + if out is not None: + body += f'out = "{out}"\n' + (project / "foundry.toml").write_text(body) + + +def _hardhat_config(project: Path) -> None: + (project / "hardhat.config.ts").write_text("export default {};") + + +def _foundry_artifacts(out_dir: Path, contract: str = "Widget", source: str = "src/Widget.sol") -> None: + """Write the artifact `forge build` leaves for one compiled contract.""" + artifact_dir = out_dir / Path(source).name + artifact_dir.mkdir(parents=True) + (artifact_dir / f"{contract}.json").write_text(json.dumps({ + "bytecode": {"object": "0x6080604052"}, + "metadata": { + "compiler": {"version": "0.8.20+commit.a1b79de6"}, + "settings": {"compilationTarget": {source: contract}}, + }, + })) + + +def _hardhat_artifacts(artifacts_dir: Path) -> None: + """Hardhat's own layout: the sources tree mirrored down to the per-contract json, plus + build-info beside it. The json matters — the mirror directories alone are created by a + configured-but-never-run build.""" + (artifacts_dir / "contracts" / "Vault.sol").mkdir(parents=True) + (artifacts_dir / "contracts" / "Vault.sol" / "Vault.json").write_text('{"abi": []}') + (artifacts_dir / "build-info").mkdir(parents=True) + (artifacts_dir / "build-info" / "1234.json").write_text("{}") + + +# --- detector: one build system ------------------------------------------------------ + + +def test_foundry_alone_is_foundry_built_or_not(tmp_path: Path) -> None: + _foundry_config(tmp_path) + + assert BuildSystemDetector.detect(tmp_path) == BuildSystem.FOUNDRY + + _foundry_artifacts(tmp_path / "out") + assert BuildSystemDetector.detect(tmp_path) == BuildSystem.FOUNDRY + + +def test_hardhat_alone_is_hardhat_built_or_not(tmp_path: Path) -> None: + _hardhat_config(tmp_path) + + assert BuildSystemDetector.detect(tmp_path) == BuildSystem.HARDHAT + + _hardhat_artifacts(tmp_path / "artifacts") + assert BuildSystemDetector.detect(tmp_path) == BuildSystem.HARDHAT + + +# --- detector: both configs present -------------------------------------------------- + + +def test_both_configs_with_foundry_artifacts_stays_foundry(tmp_path: Path) -> None: + _foundry_config(tmp_path) + _hardhat_config(tmp_path) + _foundry_artifacts(tmp_path / "out") + + assert BuildSystemDetector.detect(tmp_path) == BuildSystem.FOUNDRY + + +def test_both_configs_unbuilt_stays_foundry(tmp_path: Path) -> None: + # No evidence either way, which is every run that happens before the build. + _foundry_config(tmp_path) + _hardhat_config(tmp_path) + + assert BuildSystemDetector.detect(tmp_path) == BuildSystem.FOUNDRY + + +def test_both_configs_both_built_stays_foundry(tmp_path: Path) -> None: + _foundry_config(tmp_path) + _hardhat_config(tmp_path) + _foundry_artifacts(tmp_path / "out") + _hardhat_artifacts(tmp_path / "artifacts") + + assert BuildSystemDetector.detect(tmp_path) == BuildSystem.FOUNDRY + + +def test_both_configs_with_only_hardhat_artifacts_picks_hardhat(tmp_path: Path) -> None: + # The tie-break: a Hardhat project whose foundry.toml governs a forge test harness + # nobody built. Reading Foundry's empty out/ there yields no contracts at all. + _foundry_config(tmp_path) + _hardhat_config(tmp_path) + _hardhat_artifacts(tmp_path / "artifacts") + + assert BuildSystemDetector.detect(tmp_path) == BuildSystem.HARDHAT + + +def test_an_empty_foundry_out_is_not_evidence(tmp_path: Path) -> None: + _foundry_config(tmp_path) + _hardhat_config(tmp_path) + (tmp_path / "out").mkdir() + _hardhat_artifacts(tmp_path / "artifacts") + + assert BuildSystemDetector.detect(tmp_path) == BuildSystem.HARDHAT + + +def test_foundrys_evidence_is_read_from_its_configured_out(tmp_path: Path) -> None: + # Foundry built into a non-default directory: the artifacts are still Foundry's, so + # the tie must not go to Hardhat just because out/ is nowhere to be seen. + _foundry_config(tmp_path, out="artifacts-forge") + _hardhat_config(tmp_path) + _foundry_artifacts(tmp_path / "artifacts-forge") + _hardhat_artifacts(tmp_path / "artifacts") + + assert BuildSystemDetector.detect(tmp_path) == BuildSystem.FOUNDRY + + +def test_an_explicit_build_system_beats_the_artifacts(tmp_path: Path) -> None: + _foundry_config(tmp_path) + _hardhat_config(tmp_path) + _hardhat_artifacts(tmp_path / "artifacts") + + assert BuildSystemDetector.resolve(tmp_path, "foundry") == BuildSystem.FOUNDRY + + +# --- extractor: which directory the artifacts are read from -------------------------- + + +@pytest.fixture +def project(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path: + """A one-contract Foundry project; the source has to exist to be in scope.""" + (tmp_path / "src").mkdir() + (tmp_path / "src" / "Widget.sol").write_text("contract Widget {}") + monkeypatch.chdir(tmp_path) + return tmp_path + + +def _extracted(project: Path) -> list[str]: + return [h.contract_name for h in FoundryContractExtractor(project).extract_logic_contracts()] + + +def test_artifacts_are_read_from_the_default_out(project: Path) -> None: + _foundry_config(project) + _foundry_artifacts(project / "out") + + assert _extracted(project) == ["Widget"] + + +def test_a_configured_out_nested_under_the_default_name_is_read(project: Path) -> None: + # `out/foundry` leaves a bare `out/` that exists and holds no artifacts; reading it + # instead of the configured directory finds nothing to verify. + _foundry_config(project, out="out/foundry") + _foundry_artifacts(project / "out" / "foundry") + + assert _extracted(project) == ["Widget"] + + +def test_a_populated_default_out_wins_over_the_configured_one(project: Path) -> None: + # Foundry itself reads the profile's `out`, so the two agreeing is the normal case. When + # they disagree, the default is the one that costs nothing to look at, so it is what gets + # read; asking the config first would run `forge remappings`, or node for the other two + # build systems, on every extraction. + (project / "src" / "Gadget.sol").write_text("contract Gadget {}") + _foundry_config(project, out="artifacts-forge") + _foundry_artifacts(project / "out") + _foundry_artifacts(project / "artifacts-forge", contract="Gadget", source="src/Gadget.sol") + + assert _extracted(project) == ["Widget"] + + +def test_the_configured_out_is_read_when_the_default_out_is_empty(project: Path) -> None: + # The default directory is left behind by a build that no longer runs, so it holds + # nothing to rank it by and the config gets to name the directory instead. + _foundry_config(project, out="artifacts-forge") + (project / "out").mkdir() + _foundry_artifacts(project / "artifacts-forge") + + assert _extracted(project) == ["Widget"] + + +def test_an_empty_default_out_is_read_rather_than_reported_as_missing(project: Path) -> None: + # Nothing built anywhere, and the only directory on disk is the default one. A directory + # that is there ends the search: it is read, finds nothing, and the caller reports on that + # itself. Raising instead would name a directory nobody has to create by hand. + _foundry_config(project, out="artifacts-forge") + (project / "out").mkdir() + + assert _extracted(project) == [] + + +def test_the_error_names_the_default_when_the_config_cannot_be_read(project: Path) -> None: + # No foundry.toml to read, so the default name is the only directory we can point at. + with pytest.raises(Exception, match=r"out' does not exist"): + _extracted(project) + + +def test_a_file_where_the_artifacts_directory_belongs_is_reported_as_such(project: Path) -> None: + # A path that is there but is a file needs a different remedy from one that is absent, so + # the two are reported apart. + _foundry_config(project) + (project / "out").write_text("") + + with pytest.raises(Exception, match=r"out' is not a directory"): + _extracted(project) + + +def test_the_source_path_map_is_empty_for_an_unbuilt_project(project: Path) -> None: + # The map is built by walking the artifacts directory, so an unbuilt project has to be + # answered before the walk rather than by it. + _foundry_config(project) + + assert FoundryContractExtractor(project).build_source_path_to_contracts_map() == {} + + +def test_an_unbuilt_project_names_the_build_command(project: Path) -> None: + _foundry_config(project) + + with pytest.raises(Exception, match="forge build"): + _extracted(project) + + +def test_an_empty_hardhat_artifacts_dir_is_not_evidence(tmp_path: Path) -> None: + # Symmetry with the Foundry side: a leftover empty artifacts/contracts/ says a Hardhat build + # was configured, not that one ran, so it must not take the tie from a Foundry tree. + (tmp_path / "foundry.toml").write_text("[profile.default]\n") + (tmp_path / "hardhat.config.ts").write_text("export default {};\n") + (tmp_path / "out").mkdir() + (tmp_path / "artifacts" / "contracts").mkdir(parents=True) + + assert BuildSystemDetector.detect(tmp_path) == BuildSystem.FOUNDRY + + +def test_the_unreadable_artifacts_error_names_the_configured_directory(tmp_path: Path) -> None: + # An unbuilt project whose config points elsewhere should be told about that directory, + # not about the build system's default which it never uses. + (tmp_path / "foundry.toml").write_text('[profile.default]\nout = "out/foundry"\n') + (tmp_path / "src").mkdir() + + extractor = FoundryContractExtractor(tmp_path) + + with pytest.raises(Exception, match=r"out/foundry.*does not exist"): + extractor.extract_logic_contracts() diff --git a/tests/test_build_system_run_root.py b/tests/test_build_system_run_root.py new file mode 100644 index 00000000..82d5aaf5 --- /dev/null +++ b/tests/test_build_system_run_root.py @@ -0,0 +1,74 @@ +"""The build-config dir and the run root must reach the remapping builder as two distinct paths. + +They differ exactly for a monorepo sub-project — the case remapping-context rebasing and the +hoisted-package walk were written for — so a manager constructed with only the build-config dir +makes both of them no-ops on the projects that need them. +""" + +from pathlib import Path + +import pytest + +from certora_autosetup.build_systems.foundry import FoundryManager +from certora_autosetup.build_systems.truffle import TruffleManager +import certora_autosetup.build_systems.foundry as foundry_mod +import certora_autosetup.build_systems.truffle as truffle_mod + + +@pytest.fixture +def captured_builder_kwargs(monkeypatch: pytest.MonkeyPatch) -> dict: + """Capture the kwargs the managers pass to build_packages_from_remapping_sources.""" + captured: dict = {} + + def fake_builder(**kwargs): + captured.update(kwargs) + return [] + + monkeypatch.setattr(foundry_mod, "build_packages_from_remapping_sources", fake_builder) + monkeypatch.setattr(truffle_mod, "build_packages_from_remapping_sources", fake_builder) + return captured + + +def test_foundry_manager_forwards_both_directories(tmp_path: Path, captured_builder_kwargs) -> None: + project = tmp_path / "smart-contracts" + project.mkdir() + foundry_toml = project / "foundry.toml" + foundry_toml.write_text("[profile.default]\n") + + FoundryManager(project_root=project, scope=None, run_root=tmp_path).parse_config(foundry_toml) + + assert captured_builder_kwargs["base_dir"] == project + assert captured_builder_kwargs["run_root"] == tmp_path + + +def test_truffle_manager_forwards_both_directories(tmp_path: Path, captured_builder_kwargs) -> None: + project = tmp_path / "smart-contracts" + project.mkdir() + config_file = project / "truffle-config.js" + config_file.write_text("module.exports = {};\n") + + TruffleManager(project_root=project, scope=None, run_root=tmp_path).parse_config(config_file) + + assert captured_builder_kwargs["base_dir"] == project + assert captured_builder_kwargs["run_root"] == tmp_path + + +def test_run_root_defaults_to_the_project_root(tmp_path: Path, captured_builder_kwargs) -> None: + # A project whose build config sits at the run root needs no caller change. + foundry_toml = tmp_path / "foundry.toml" + foundry_toml.write_text("[profile.default]\n") + + FoundryManager(project_root=tmp_path, scope=None).parse_config(foundry_toml) + + assert captured_builder_kwargs["run_root"] == tmp_path + + +def test_every_manager_class_accepts_a_run_root(tmp_path: Path) -> None: + # autosetup constructs whichever class detection picked through one polymorphic call, so a + # manager that does not accept run_root would raise TypeError at setup time. + from certora_autosetup.build_systems.hardhat import HardhatManager + + for manager_class in (FoundryManager, HardhatManager, TruffleManager): + manager = manager_class(tmp_path / "sub", None, run_root=tmp_path) + assert manager.run_root == tmp_path + assert manager.project_root == tmp_path / "sub" diff --git a/tests/test_cex_analysis_failure_isolation.py b/tests/test_cex_analysis_failure_isolation.py new file mode 100644 index 00000000..789f5b2a --- /dev/null +++ b/tests/test_cex_analysis_failure_isolation.py @@ -0,0 +1,93 @@ +"""A failing per-CEX analysis must cost only that counterexample's explanation. + +``TrivialFanoutCexHandler.analyze`` fans the violated rules out concurrently. The +fan-out used to be a bare ``asyncio.gather``, so the first analysis to raise +propagated out of ``analyze`` → ``run_prover`` → the pipeline and ended the run — +after the prover had already done its work. A transient API error on one +counterexample was enough to discard hours of verification. + +These tests pin the isolation: the surviving rules keep their explanations, the +report still renders, and the summarization threshold is unmoved by whether an +analysis succeeded or blew up. +""" + +import asyncio + +import pytest + +from composer.prover import core +from composer.prover.core import TrivialFanoutCexHandler +from composer.prover.ptypes import RulePath, RuleResult + + +def _violated(rule: str) -> RuleResult: + return RuleResult(path=RulePath(rule=rule), cex_dump=f"", status="VIOLATED") + + +class _RecordingCallbacks: + def __init__(self) -> None: + self.started: list[str] = [] + self.completed: dict[str, str] = {} + + async def on_analysis_start(self, rule: RuleResult) -> None: + self.started.append(rule.name) + + async def on_analysis_complete(self, rule: RuleResult, explanation: str) -> None: + self.completed[rule.name] = explanation + + +def handler_for() -> TrivialFanoutCexHandler: + # ``llm`` is only reached through the patched ``analyze_cex_raw`` (and by + # ``_report_to_todo_list``, which these cases stay under the threshold of). + return TrivialFanoutCexHandler(llm=None, state={"messages": []}) # type: ignore[arg-type] + + +async def _analyze(handler: TrivialFanoutCexHandler, rules: list[RuleResult], tmp_path): + callbacks = _RecordingCallbacks() + report = await handler.analyze(rules, "tool-call-1", callbacks, tmp_path) # type: ignore[arg-type] + return report, callbacks + + +@pytest.mark.asyncio +async def test_one_failed_analysis_does_not_sink_the_others(monkeypatch, tmp_path) -> None: + async def fake_analyze(llm, messages, rule: RuleResult, tool_call_id: str) -> str: + if rule.name == "boom": + raise RuntimeError("api blew up") + return f"explanation for {rule.name}" + + monkeypatch.setattr(core, "analyze_cex_raw", fake_analyze) + + rules = [_violated("first"), _violated("boom"), _violated("second")] + report, callbacks = await _analyze(handler_for(), rules, tmp_path) + + assert "explanation for first" in report + assert "explanation for second" in report + assert set(callbacks.completed) == {"first", "second"} + + +@pytest.mark.asyncio +async def test_every_analysis_failing_still_renders_a_report(monkeypatch, tmp_path) -> None: + async def fake_analyze(llm, messages, rule: RuleResult, tool_call_id: str) -> str: + raise RuntimeError("api blew up") + + monkeypatch.setattr(core, "analyze_cex_raw", fake_analyze) + + rules = [_violated("first"), _violated("second")] + report, callbacks = await _analyze(handler_for(), rules, tmp_path) + + # The rules' statuses come from the prover, not from their analyses, so the + # report still has something to say about them. + assert "first" in report + assert "second" in report + assert callbacks.completed == {} + + +@pytest.mark.asyncio +async def test_cancellation_is_not_swallowed(monkeypatch, tmp_path) -> None: + async def fake_analyze(llm, messages, rule: RuleResult, tool_call_id: str) -> str: + raise asyncio.CancelledError() + + monkeypatch.setattr(core, "analyze_cex_raw", fake_analyze) + + with pytest.raises(asyncio.CancelledError): + await _analyze(handler_for(), [_violated("first")], tmp_path) diff --git a/tests/test_code_explorer_prompts.py b/tests/test_code_explorer_prompts.py new file mode 100644 index 00000000..ee21b1a7 --- /dev/null +++ b/tests/test_code_explorer_prompts.py @@ -0,0 +1,108 @@ +"""The explorer system prompt is per-ecosystem: shared protocol, chain look-fors.""" + +import re +from pathlib import Path + +import pytest + +from composer.pipeline.ecosystem import EVM, SOLANA, SOROBAN +from composer.spec.code_explorer import ( + CodeExplorerPromptParams, + PriorFindingsMode, + code_explorer_sys_prompt, +) +from composer.spec.gen_types import TypedTemplate +from composer.templates.loader import load_jinja_template + + +PROTOCOL = "deliver your answer via the `result` tool" +DO_NOT_GUESS = "*DO NOT GUESS*" +ESTABLISHED = "established facts" +VERSIONED = "(potentially) out of date" + +_MODES: tuple[PriorFindingsMode, ...] = ("none", "established", "versioned") + + +def _render( + template: TypedTemplate[CodeExplorerPromptParams], mode: PriorFindingsMode +) -> str: + return code_explorer_sys_prompt(template, mode)(load_jinja_template) + + +@pytest.mark.parametrize("ecosystem", [EVM, SOLANA, SOROBAN], ids=["evm", "solana", "soroban"]) +@pytest.mark.parametrize("mode", _MODES) +def test_shared_protocol(ecosystem, mode: PriorFindingsMode): + text = _render(ecosystem.code_explorer_prompt, mode) + assert PROTOCOL in text + assert DO_NOT_GUESS in text + assert (ESTABLISHED in text) is (mode == "established") + assert (VERSIONED in text) is (mode == "versioned") + + +def test_evm_cites_solidity_not_chain_lookfors(): + text = _render(EVM.code_explorer_prompt, "none") + assert "function signatures" in text + assert "state variable" in text + assert "PDA" not in text + assert "require_auth" not in text + + +def test_solana_cites_pdas_not_soroban_auth(): + text = _render(SOLANA.code_explorer_prompt, "none") + assert "Cargo.toml" in text + assert "PDA" in text + assert "CPI" in text + assert "require_auth" not in text + + +# Terms that belong on an ecosystem (or language) template, not in the shared +# explorer protocol. ``contract``/``program`` are the usual leaks: EVM/Soroban +# say one, Solana the other. +_CHAIN_TERMS = re.compile( + r"\b(solidity|evm|solana|soroban|anchor|pda|cpi|require_auth|" + r"msg\.sender|modifier|contract|program|instruction|signer|account)s?\b", + re.IGNORECASE, +) + +_SHARED_EXPLORER_DIR = Path(__file__).resolve().parent.parent / "composer" / "templates" / "code_explorer" +_SHARED_EXPLORER_TEMPLATES = ( + "common_fragment.j2", + "index_addendum_fragment.j2", + "versioned_index_addendum_fragment.j2", + "prior_findings_fragment.j2", + "rust/common_fragment.j2", +) + + +@pytest.mark.parametrize("rel", _SHARED_EXPLORER_TEMPLATES) +def test_shared_explorer_templates_are_chain_neutral(rel: str): + text = (_SHARED_EXPLORER_DIR / rel).read_text() + hits = sorted({m.group(0).lower() for m in _CHAIN_TERMS.finditer(text)}) + assert hits == [], f"{rel} leaks chain terminology: {hits}" + + +def test_soroban_cites_auth_and_storage_kind_not_pdas(): + text = _render(SOROBAN.code_explorer_prompt, "none") + assert "Cargo.toml" in text + assert "require_auth" in text + assert "DataKey" in text + assert "PDA" not in text + + +@pytest.mark.parametrize( + "template,language", + [ + ("application_analysis_system.j2", "Solidity"), + ("solana/analysis_system.j2", "Rust"), + ("solana/property_system.j2", "Rust"), + ("soroban/analysis_system.j2", "Rust"), + ("soroban/property_system.j2", "Rust"), + ], +) +def test_caller_guidance_names_the_source_language(template: str, language: str): + text = load_jinja_template( + template, sort="existing", has_doc=False, backend_guidance="" + ) + assert f"The {language} language itself" in text + if language != "Solidity": + assert "The Solidity language itself" not in text diff --git a/tests/test_compilation_workarounds.py b/tests/test_compilation_workarounds.py index 038f0dfb..7ab643e2 100644 --- a/tests/test_compilation_workarounds.py +++ b/tests/test_compilation_workarounds.py @@ -6,7 +6,12 @@ import pytest import certora_autosetup.utils.remappings as remappings_mod -from certora_autosetup.utils.compilation_workarounds import CompilationWorkaroundManager +from certora_autosetup.utils.compilation_workarounds import ( + VIA_IR_SCENE_THRESHOLD, + CompilationWorkaroundManager, + UnimplementedContractError, + UnsatisfiableSolcPinError, +) from certora_autosetup.utils.types import ContractHandle @@ -66,6 +71,12 @@ def manager(tmp_path: Path) -> CompilationWorkaroundManager: return CompilationWorkaroundManager(project_root=tmp_path) +@pytest.fixture +def manager_declaring_via_ir(tmp_path: Path) -> CompilationWorkaroundManager: + """A manager for a project whose own build config declares via_ir.""" + return CompilationWorkaroundManager(project_root=tmp_path, declared_via_ir=True) + + def test_detects_wrapped_yul_stack_too_deep(manager: CompilationWorkaroundManager) -> None: # Regression: before the DOTALL/\s+ fix the wrapped phrase was missed, so # yul_exception_add_optimizer never fired and the run died as "no applicable @@ -187,20 +198,21 @@ def test_unnamed_return_warning_fires_once(manager, monkeypatch, tmp_path) -> No def test_noop_pass_exits_without_recompile(manager, monkeypatch, tmp_path) -> None: - # Run 1: via-ir applies for Foo. Run 2: the identical stack-too-deep hit - # fires again, re-applying is a no-op; the catch-all is suppressed because - # a specific workaround applied this pass, and the pass changed nothing -> - # exit without recompiling. + # Run 1: the optimizer goes on, legacy codegen kept. Run 2: the same stack-too-deep + # survives it, so via-ir applies for Foo. Run 3: the identical hit fires again, + # re-applying is a no-op; the catch-all is suppressed because a specific workaround + # applied this pass, and the pass changed nothing -> exit without recompiling. contracts = [ContractHandle(contract_name="Foo", source_file="contracts/Foo.sol")] success, updated, _, fake_run = _run_loop_with_output( manager, monkeypatch, tmp_path, PERSISTENT_STACK_TOO_DEEP_OUTPUT, contracts ) assert success is False + assert updated["solc_optimize"] == "200" # The first application is preserved (uniform one-contract map collapses # back to the scalar on exit). assert updated["solc_via_ir"] is True assert "use_relpaths_for_solc_json" not in updated - assert fake_run.calls == 2 + assert fake_run.calls == 3 def test_different_detect_results_keep_workaround_enabled(manager, monkeypatch, tmp_path) -> None: @@ -215,11 +227,12 @@ def test_different_detect_results_keep_workaround_enabled(manager, monkeypatch, manager, monkeypatch, tmp_path, - [PERSISTENT_STACK_TOO_DEEP_OUTPUT, STACK_TOO_DEEP_BAR_OUTPUT], + [PERSISTENT_STACK_TOO_DEEP_OUTPUT, PERSISTENT_STACK_TOO_DEEP_OUTPUT, STACK_TOO_DEEP_BAR_OUTPUT], contracts, ) assert success is True - assert fake_run.calls == 3 + # Pass 1 spends the optimizer rung; Foo and Bar then need one via-ir application each. + assert fake_run.calls == 4 # Both contracts got via-ir, so the uniform map collapsed to the scalar on # exit. A guard that disabled the workaround after its first application # would leave Bar's entry False and the map uncollapsed. @@ -259,7 +272,9 @@ def test_multiple_workarounds_apply_in_one_pass(manager, monkeypatch, tmp_path) ) assert success is True assert fake_run.calls == 2 - assert updated["solc_via_ir"] is True + # The stack-too-deep half is answered by the optimizer first — via-ir is a later rung. + assert updated["solc_optimize"] == "200" + assert "solc_via_ir" not in updated assert updated["ignore_solidity_warnings"] is True assert "use_relpaths_for_solc_json" not in updated @@ -369,6 +384,151 @@ def test_via_ir_added_out_of_necessity(manager, monkeypatch, tmp_path) -> None: assert updated["solc_via_ir"] is True +# A second solc phrasing for the same condition — legacy codegen cannot do this copy, +# the IR pipeline can — reported by a whole-project compile: no "Compiling ..." +# progress line, so the file is named only in the `-->` source-location line. solc +# hard-wraps both the "IR pipeline" phrase and the `--via-ir` flag token itself. +BULK_VIA_IR_LEGACY_COPY = ( + "solc8.28 had an error:\n" + "UnimplementedFeatureError: Copying of type struct Vault.RateTier memory[] memory \n" + "to storage is not supported in legacy (only supported by the IR \n" + "pipeline). Hint: try compiling with `--via-\n" + "ir` (CLI) or the equivalent `viaIR: true` (Standard JSON)\n" + " --> contracts/Vault.sol:120:9:\n" +) + +# The error calling for the OPPOSITE fix (turn via-ir OFF for an old compiler). It +# names the conf key solc_via_ir, which must stay outside the via-ir-required family. +UNSUPPORTED_SOLC_VIA_IR_OUTPUT = ( + "Compiling contracts/Foo.sol...\n" + "Unsupported solc version 0.7.6 for solc_via_ir, please use 0.8.13 or later\n" +) + +# A per-unit error: the "Compiling ..." line names Foo while the `-->` names an +# unrelated inlined file. +COMPILING_LINE_VIA_IR_REQUIRED = ( + "Compiling contracts/Foo.sol...\n" + "solc8.26 had an error:\n" + "UnimplementedFeatureError: Require with a custom error is only available using \n" + "the via-ir pipeline.\n" + " --> lib/somewhere/Inlined.sol:10:5:\n" +) + + +# Whole-project compile where an unrelated Warning with its own source location follows +# the via-ir diagnostic, which names no file of its own. +BULK_VIA_IR_FOLLOWED_BY_FOREIGN_WARNING = ( + "solc8.28 had an error:\n" + "UnimplementedFeatureError: Copying of type struct Vault.RateTier memory[] memory \n" + "to storage is not supported in legacy (only supported by the IR pipeline).\n" + "Warning: Unused function parameter. Remove or comment out the variable name.\n" + " --> lib/oz/ERC20.sol:80:5:\n" +) + +# Each of the three hint spellings on its own, so no single fixture can cover for a +# broken alternative. The diagnostic wording is the same in all three and mentions +# neither the pipeline nor the flag. +VIA_IR_HINT_FLAG_ONLY = ( + "Compiling contracts/Foo.sol...\n" + "solc8.26 had an error:\n" + "UnimplementedFeatureError: This feature is not supported by the legacy code \n" + "generator. Hint: try compiling with `--via-ir` (CLI).\n" +) + +# The flag token itself split by solc's hard wrap. +VIA_IR_HINT_FLAG_WRAPPED = ( + "Compiling contracts/Foo.sol...\n" + "solc8.26 had an error:\n" + "UnimplementedFeatureError: This feature is not supported by the legacy code \n" + "generator. Hint: try compiling with `--via-\n" + "ir` (CLI).\n" +) + +VIA_IR_HINT_JSON_KEY_ONLY = ( + "Compiling contracts/Foo.sol...\n" + "solc8.26 had an error:\n" + "UnimplementedFeatureError: This feature is not supported by the legacy code \n" + "generator. Hint: set `viaIR: true` (Standard JSON).\n" +) + +# A diagnostic whose quoted source line happens to talk about a pipeline: prose in the +# user's code, not a solc remediation hint. +TYPE_ERROR_WITH_PIPELINE_PROSE = ( + "Compiling contracts/Foo.sol...\n" + "solc8.26 had an error:\n" + 'TypeError: Member "route" not found or not visible after argument-dependent lookup.\n' + " --> contracts/Foo.sol:42:9:\n" + " |\n" + " 42 | router.route(amount); // route through their pipeline\n" +) + + +def test_detects_bulk_via_ir_required_via_source_location(manager) -> None: + # Whole-project compile: the affected file comes from `--> :line:col`, and the + # wrapped `--via-\nir` hint must still be recognized. + contracts = [ContractHandle(contract_name="Vault", source_file="contracts/Vault.sol")] + assert manager._detect_via_ir_required(BULK_VIA_IR_LEGACY_COPY, contracts) == "Vault" + + +def test_unsupported_solc_via_ir_is_not_via_ir_required(manager) -> None: + # Enabling via-ir here would fight the workaround that must disable it. + contracts = [ContractHandle(contract_name="Foo", source_file="contracts/Foo.sol")] + assert manager._detect_via_ir_required(UNSUPPORTED_SOLC_VIA_IR_OUTPUT, contracts) is None + + +def test_via_ir_compiling_line_takes_precedence_over_source_location(manager) -> None: + # The compiled unit is what needs via-ir; the inlined file in the `-->` line is a + # scene contract too, so only precedence decides the answer. + contracts = [ + ContractHandle(contract_name="Foo", source_file="contracts/Foo.sol"), + ContractHandle(contract_name="Inlined", source_file="lib/somewhere/Inlined.sol"), + ] + assert manager._detect_via_ir_required(COMPILING_LINE_VIA_IR_REQUIRED, contracts) == "Foo" + + +def test_via_ir_fallback_ignores_another_diagnostics_source_location(manager) -> None: + # The `-->` belongs to the Warning below, not to the via-ir diagnostic — enabling + # via-ir for that file would fix nothing and change an unrelated contract's build. + contracts = [ContractHandle(contract_name="ERC20", source_file="lib/oz/ERC20.sol")] + assert ( + manager._detect_via_ir_required(BULK_VIA_IR_FOLLOWED_BY_FOREIGN_WARNING, contracts) is None + ) + + +@pytest.mark.parametrize( + "output", + [VIA_IR_HINT_FLAG_ONLY, VIA_IR_HINT_FLAG_WRAPPED, VIA_IR_HINT_JSON_KEY_ONLY], + ids=["flag", "flag-wrapped", "json-key"], +) +def test_each_hint_spelling_detected_on_its_own(manager, output: str) -> None: + # One spelling per fixture: a broken alternative cannot hide behind another one + # matching first. + contracts = [ContractHandle(contract_name="Foo", source_file="contracts/Foo.sol")] + assert manager._detect_via_ir_required(output, contracts) == "Foo" + + +def test_pipeline_prose_in_source_line_is_not_a_hint(manager) -> None: + contracts = [ContractHandle(contract_name="Foo", source_file="contracts/Foo.sol")] + assert manager._detect_via_ir_required(TYPE_ERROR_WITH_PIPELINE_PROSE, contracts) is None + + +def test_stack_too_deep_hint_is_not_via_ir_required(manager) -> None: + # solc appends the same via-ir hint to every stack-too-deep / YulException + # diagnostic, where via-ir is one remedy among several. Those must stay with + # stack_too_deep_via_ir and the yul rungs, which climb the optimizer ladder first. + foo = [ContractHandle(contract_name="Foo", source_file="contracts/Foo.sol")] + harness = [ + ContractHandle( + contract_name="LMPStrategyInstance1", + source_file="certora/harnesses/LMPStrategyInstance1.sol", + ) + ] + assert manager._detect_via_ir_required(WRAPPED_YUL_STACK_TOO_DEEP, harness) is None + assert manager._detect_via_ir_required(SINGLE_LINE_YUL_STACK_TOO_DEEP, foo) is None + assert manager._detect_via_ir_required(PERSISTENT_STACK_TOO_DEEP_OUTPUT, foo) is None + assert manager._detect_via_ir_required(BULK_STACK_TOO_DEEP, foo) is None + + def test_yul_last_resort_keeps_compile_settings(manager, monkeypatch, tmp_path) -> None: # Pass 1 carries a plain stack-too-deep for Foo AND a YulException with the # optimizer already present (e.g. supplied by the project's foundry config): @@ -603,7 +763,9 @@ def test_solc_fallback_fires_when_pin_is_in_compiler_map( # The pin can arrive already folded into compiler_map with no scalar "solc" # (precomputed from build artifacts) — the missing-binary fallback must # still be armed, keyed on the map contents rather than the scalar. - monkeypatch.setattr(manager, "_pick_solc_fallback", lambda: "solc8.30") + # No source file on disk here, so the pragma is unreadable and the substitution + # is taken on the first candidate. + monkeypatch.setattr(manager, "_solc_fallback_candidates", lambda: [("solc8.30", "0.8.30")]) contracts = [ContractHandle(contract_name="Vault", source_file="contracts/Vault.sol")] success, _, compilation_config, fake_run = _run_loop( manager, @@ -620,6 +782,316 @@ def test_solc_fallback_fires_when_pin_is_in_compiler_map( assert "compiler_map" not in compilation_config +def _write_pragma(tmp_path, contract: str, pragma: str) -> ContractHandle: + source = tmp_path / "contracts" / f"{contract}.sol" + source.parent.mkdir(parents=True, exist_ok=True) + source.write_text(f"pragma solidity {pragma};\ncontract {contract} {{}}\n") + return ContractHandle(contract_name=contract, source_file=f"contracts/{contract}.sol") + + +def test_solc_fallback_refused_when_it_contradicts_an_exact_pragma( + manager, monkeypatch, tmp_path +) -> None: + # An exact pragma admits exactly one compiler. Substituting any other reproduces + # the same ParserError next pass, so the run stops and names what to install. + monkeypatch.setattr(manager, "_solc_fallback_candidates", lambda: [("solc8.34", "0.8.34")]) + contracts = [_write_pragma(tmp_path, "Vault", "0.6.4")] + with pytest.raises(UnsatisfiableSolcPinError) as excinfo: + _run_loop( + manager, + monkeypatch, + tmp_path, + [SOLC_NOT_FOUND_OUTPUT.replace("solc8.35", "solc6.4")], + contracts, + extra_config={"compiler_map": {"Vault": "solc6.4"}}, + ) + message = str(excinfo.value) + assert "Vault" in message and "0.6.4" in message and "solc6.4" in message + + +def test_satisfiable_range_pragma_still_falls_back(manager, monkeypatch, tmp_path) -> None: + # A range pragma the installed compiler satisfies must keep substituting. + monkeypatch.setattr(manager, "_solc_fallback_candidates", lambda: [("solc8.34", "0.8.34")]) + contracts = [_write_pragma(tmp_path, "Vault", "^0.8.0")] + success, _, compilation_config, fake_run = _run_loop( + manager, + monkeypatch, + tmp_path, + [SOLC_NOT_FOUND_OUTPUT], + contracts, + extra_config={"compiler_map": {"Vault": "solc8.35"}}, + ) + assert success is True + assert fake_run.calls == 2 + assert compilation_config["solc"] == "solc8.34" + + +def test_unparseable_pragma_is_not_treated_as_a_contradiction( + manager, monkeypatch, tmp_path +) -> None: + # A disjunction cannot be expressed as one SpecifierSet, so the resolver returns + # no constraint. Unknown is not a conflict, and the substitution proceeds. + monkeypatch.setattr(manager, "_solc_fallback_candidates", lambda: [("solc8.34", "0.8.34")]) + contracts = [_write_pragma(tmp_path, "Vault", "^0.6.0 || ^0.7.0")] + success, _, compilation_config, _ = _run_loop( + manager, + monkeypatch, + tmp_path, + [SOLC_NOT_FOUND_OUTPUT], + contracts, + extra_config={"compiler_map": {"Vault": "solc8.35"}}, + ) + assert success is True + assert compilation_config["solc"] == "solc8.34" + + +def test_fallback_is_decided_per_contract(manager, monkeypatch, tmp_path) -> None: + # One contract can be served and another cannot; the blocked one is named and + # the satisfiable one is not blamed. + monkeypatch.setattr(manager, "_solc_fallback_candidates", lambda: [("solc8.34", "0.8.34")]) + contracts = [ + _write_pragma(tmp_path, "Vault", "^0.8.0"), + _write_pragma(tmp_path, "Legacy", "0.6.4"), + ] + with pytest.raises(UnsatisfiableSolcPinError) as excinfo: + _run_loop( + manager, + monkeypatch, + tmp_path, + [SOLC_NOT_FOUND_OUTPUT], + contracts, + extra_config={"compiler_map": {"Vault": "solc8.35", "Legacy": "solc8.35"}}, + ) + message = str(excinfo.value) + assert "Legacy" in message + assert "Vault requires" not in message + + +def test_project_default_is_preferred_over_whatever_solc_is_on_path( + manager, monkeypatch, tmp_path +) -> None: + # A wide pragma admits both, and plain `solc` may be any unrelated version the + # machine happens to carry, so the project's own default wins. + monkeypatch.setattr(manager, "_get_plain_solc_version", lambda: "0.5.16") + monkeypatch.setattr( + "certora_autosetup.utils.compilation_workarounds.shutil.which", lambda _: "/usr/bin/solc8.34" + ) + monkeypatch.setattr(manager, "solc_default_version", "solc8.34") + contracts = [_write_pragma(tmp_path, "Vault", ">=0.4.22 <0.9.0")] + success, _, compilation_config, _ = _run_loop( + manager, + monkeypatch, + tmp_path, + [SOLC_NOT_FOUND_OUTPUT], + contracts, + extra_config={"compiler_map": {"Vault": "solc8.35"}}, + ) + assert success is True + assert compilation_config["solc"] == "solc8.34" + + +def test_candidates_fall_through_to_the_next_when_the_pragma_rejects_the_first( + manager, monkeypatch, tmp_path +) -> None: + # Per-contract selection walks the candidate list rather than taking the head. + monkeypatch.setattr( + manager, "_solc_fallback_candidates", lambda: [("solc8.34", "0.8.34"), ("solc6.12", "0.6.12")] + ) + contracts = [_write_pragma(tmp_path, "Legacy", "^0.6.0")] + success, _, compilation_config, _ = _run_loop( + manager, + monkeypatch, + tmp_path, + [SOLC_NOT_FOUND_OUTPUT], + contracts, + extra_config={"compiler_map": {"Legacy": "solc8.35"}}, + ) + assert success is True + assert compilation_config["solc"] == "solc6.12" + + +def test_a_source_that_is_not_utf8_reads_as_an_unknown_pragma( + manager, monkeypatch, tmp_path +) -> None: + # An accented byte in a header comment must not abort the loop. + monkeypatch.setattr(manager, "_solc_fallback_candidates", lambda: [("solc8.34", "0.8.34")]) + source = tmp_path / "contracts" / "Vault.sol" + source.parent.mkdir(parents=True, exist_ok=True) + source.write_bytes(b"// auteur: Fran\xe7ois\npragma solidity 0.6.4;\ncontract Vault {}\n") + contracts = [ContractHandle(contract_name="Vault", source_file="contracts/Vault.sol")] + success, _, compilation_config, _ = _run_loop( + manager, + monkeypatch, + tmp_path, + [SOLC_NOT_FOUND_OUTPUT], + contracts, + extra_config={"compiler_map": {"Vault": "solc8.35"}}, + ) + assert success is True + assert compilation_config["solc"] == "solc8.34" + + +def test_a_pin_autosetup_seeded_itself_is_not_terminal(manager, monkeypatch, tmp_path) -> None: + # With no pin in the conf the seeding assigns the default compiler; refusing to + # proceed over that would fail a run the user never constrained. + monkeypatch.setattr(manager, "_solc_fallback_candidates", lambda: []) + contracts = [_write_pragma(tmp_path, "Vault", "0.6.4")] + success, _, _, _ = _run_loop( + manager, monkeypatch, tmp_path, [SOLC_NOT_FOUND_OUTPUT] * 4, contracts + ) + assert success is False + + +def test_the_seen_state_memo_is_scoped_to_one_loop_not_to_the_manager( + manager, monkeypatch, tmp_path +) -> None: + # fixconf runs the loop twice on one manager, either side of the import patch; + # the second run must be free to revisit the states the first one reached. + monkeypatch.setattr(manager, "_solc_fallback_candidates", lambda: [("solc8.34", "0.8.34")]) + contracts = [ContractHandle(contract_name="Foo", source_file="contracts/Foo.sol")] + for _ in range(2): + success, _, _, fake_run = _run_loop( + manager, + monkeypatch, + tmp_path, + [MISSING_PIN_OUTPUT, PIN_DEMANDED_AGAIN_OUTPUT] * 5, + contracts, + extra_config={"compiler_map": {"Foo": "solc6.4"}}, + ) + assert success is False + assert fake_run.calls == 2 + + +def test_no_installed_candidate_blocks_instead_of_guessing( + manager, monkeypatch, tmp_path +) -> None: + # With nothing installed there is no substitution to defend, so the run stops + # rather than pinning a compiler that is equally absent. + monkeypatch.setattr(manager, "_solc_fallback_candidates", lambda: []) + contracts = [_write_pragma(tmp_path, "Vault", "^0.8.0")] + with pytest.raises(UnsatisfiableSolcPinError): + _run_loop( + manager, + monkeypatch, + tmp_path, + [SOLC_NOT_FOUND_OUTPUT], + contracts, + extra_config={"compiler_map": {"Vault": "solc8.35"}}, + ) + + +# The two halves of a real cycle. A contract is pinned to a compiler that is not +# installed; the fallback substitutes the default one; the next compile reports the +# pragma mismatch and pins the missing compiler again. The two fire on mutually +# exclusive outputs, so they land in different passes and every pass changes the +# conf — only the states the loop has already compiled reveal the cycle. +MISSING_PIN_OUTPUT = ( + "attribute/flag 'compiler_map': Solidity executable solc6.4 not found in path\n" +) + +PIN_DEMANDED_AGAIN_OUTPUT = ( + "Compiling contracts/Foo.sol...\n" + "solc8.34 had an error:\n" + "contracts/Foo.sol:1:1: ParserError: Source file requires different compiler version " + "(current compiler is 0.8.34+commit.aaaaaaaa.Linux.g++)\n" + "pragma solidity 0.6.4;\n" +) + + +def test_a_conf_state_seen_before_stops_the_loop(manager, monkeypatch, tmp_path) -> None: + # Foo's source is not on disk, so its pragma is unreadable and the fallback plan + # has no constraint to refuse on — the substitution is allowed and the cycle is + # reachable. This is the residual case the seen-state memo exists for. + monkeypatch.setattr(manager, "_solc_fallback_candidates", lambda: [("solc8.34", "0.8.34")]) + contracts = [ContractHandle(contract_name="Foo", source_file="contracts/Foo.sol")] + success, _, _, fake_run = _run_loop( + manager, + monkeypatch, + tmp_path, + [MISSING_PIN_OUTPUT, PIN_DEMANDED_AGAIN_OUTPUT] * 5, + contracts, + extra_config={"compiler_map": {"Foo": "solc6.4"}}, + ) + assert success is False + # Substitute, then re-pin — which lands back on the conf the first compile ran on. + assert fake_run.calls == 2 + + +# A missing-library link error, one per consumer. The harness workaround reacts by +# generating a harness source on disk and swapping the consumer for it in `files`. +def _missing_library_output(consumer_path: str, consumer: str, lib: str, lib_path: str) -> str: + return ( + f"Compiling {consumer_path}...\n" + f"Failed to find a dependency library while building the constructor bytecode of {consumer}.\n" + f"Failed to find a contract named {lib} in file {lib_path}.\n" + ) + + +def test_the_harness_workaround_is_not_stopped_by_the_seen_state_memo( + manager, monkeypatch, tmp_path +) -> None: + # Part of this workaround's progress is the generated harness source, which the + # memo cannot see. What it can see is the `files` swap the same apply performs, + # and that reaches a new state on every firing — so consecutive firings must not + # be read as a cycle. + (tmp_path / "contracts").mkdir(parents=True, exist_ok=True) + for lib in ("LibA", "LibB"): + (tmp_path / "contracts" / f"{lib}.sol").write_text( + f"pragma solidity ^0.8.0;\nlibrary {lib} {{\n" + f" function value() public pure returns (uint256) {{ return 1; }}\n}}\n" + ) + contracts = [] + for name in ("Foo", "Bar"): + (tmp_path / "contracts" / f"{name}.sol").write_text( + f"pragma solidity ^0.8.0;\ncontract {name} {{\n" + f" function ping() external pure returns (uint256) {{ return 1; }}\n}}\n" + ) + contracts.append( + ContractHandle(contract_name=name, source_file=f"contracts/{name}.sol") + ) + + success, _, compilation_config, fake_run = _run_loop( + manager, + monkeypatch, + tmp_path, + [ + _missing_library_output("contracts/Foo.sol", "Foo", "LibA", "contracts/LibA.sol"), + _missing_library_output("contracts/Bar.sol", "Bar", "LibB", "contracts/LibB.sol"), + ], + contracts, + ) + assert success is True + # Both firings landed, and the third compile is the one that succeeds. + assert fake_run.calls == 3 + files = compilation_config["files"] + assert any("FooHarness" in entry for entry in files) + assert any("BarHarness" in entry for entry in files) + + +def test_a_new_change_alongside_a_repeat_keeps_going(manager, monkeypatch, tmp_path) -> None: + # The same cycle, but the second pass also lands an orthogonal change. That takes + # the loop to a conf it has not compiled, so it must keep going — and it still + # terminates one full turn of the cycle later. + monkeypatch.setattr(manager, "_solc_fallback_candidates", lambda: [("solc8.34", "0.8.34")]) + contracts = [ContractHandle(contract_name="Foo", source_file="contracts/Foo.sol")] + success, _, compilation_config, fake_run = _run_loop( + manager, + monkeypatch, + tmp_path, + [ + MISSING_PIN_OUTPUT, + PIN_DEMANDED_AGAIN_OUTPUT + UNNAMED_RETURN_WARNING_OUTPUT, + MISSING_PIN_OUTPUT, + PIN_DEMANDED_AGAIN_OUTPUT, + ], + contracts, + extra_config={"compiler_map": {"Foo": "solc6.4"}}, + ) + assert success is False + assert fake_run.calls == 4 + assert compilation_config.get("ignore_solidity_warnings") is True + + def test_yul_optimizer_rung_respects_project_optimize_map( manager, monkeypatch, tmp_path ) -> None: @@ -904,3 +1376,327 @@ def test_unseeded_cancun_map_not_promoted_to_scalar(manager, monkeypatch, tmp_pa assert success is True assert compilation_config["solc_evm_version_map"] == {"Foo": "cancun"} assert "solc_evm_version" not in compilation_config + + +# ============================================================================= +# Unresolved-import classification around the source-not-found workaround +# ============================================================================= +# +# The classification never gates the workaround: it explains what the rebuild is reacting to, +# and turns the loop's generic "conf and command are unchanged" giving-up message into one that +# names why nothing could change. + + +class _SequencedRunWithoutForge(_SequencedRun): + """Queued compilation outputs, plus `forge remappings` failing the way it does in CI. + + The packages rebuild shells out to forge through the same subprocess module the loop's fake + is installed on, so one fake has to answer both kinds of call. + """ + + def __call__(self, cmd, **kwargs): + if cmd and cmd[0] == "forge": + raise FileNotFoundError("forge") + return super().__call__(cmd, **kwargs) + + +def _run_loop_with_packages(manager, monkeypatch, tmp_path, outputs, contracts, packages): + """Like _run_loop, but the conf already carries a packages list (in both dicts), which is + what a real run looks like once the build system contributed one.""" + fake_run = _SequencedRunWithoutForge(outputs) + monkeypatch.setattr( + "certora_autosetup.utils.compilation_workarounds.subprocess.run", fake_run + ) + compilation_config = { + "files": [f"{c.source_file}:{c.contract_name}" for c in contracts], + "packages": list(packages), + } + success, _, updated = manager.run_compilation_with_workarounds( + cmd=["certoraRun", "test.conf"], + config_file=tmp_path / "test.conf", + compilation_config=compilation_config, + contracts=contracts, + updated_config_dict={"packages": list(packages)}, + ) + return success, updated, fake_run + + +def _source_not_found(source_unit: str) -> str: + return f'ParserError: Source "{source_unit}" not found: File not found.\n' + + +def test_file_missing_in_installed_package_is_named_when_giving_up( + tmp_path: Path, monkeypatch +) -> None: + # The package IS installed, so the rebuilt list is identical and the loop stops — the + # message must say that rebuilding cannot help rather than only "nothing changed". + (tmp_path / "node_modules" / "@vault" / "core").mkdir(parents=True) + (tmp_path / "remappings.txt").write_text("@vault/=node_modules/@vault/core/\n") + packages = [f"@vault/={tmp_path / 'node_modules/@vault/core'}/"] + manager = CompilationWorkaroundManager(project_root=tmp_path) + logged: list[tuple[str, str]] = [] + monkeypatch.setattr(manager, "log", lambda msg, level="INFO": logged.append((msg, level))) + contracts = [ContractHandle(contract_name="Widget", source_file="src/Widget.sol")] + + success, _, fake_run = _run_loop_with_packages( + manager, + monkeypatch, + tmp_path, + [_source_not_found(f"{tmp_path / 'node_modules/@vault/core/IVault.sol'}")] * 10, + contracts, + packages, + ) + + assert success is False + # One certoraRun: the rebuild reproduces the identical list, so there is nothing to retry. + assert fake_run.calls == 1 + assert [f.kind.value for f in manager.last_import_diagnostics] == ["file_missing_in_package"] + errors = [msg for msg, level in logged if level == "ERROR"] + assert any("rebuilding the packages list cannot help" in msg for msg in errors) + + +def test_missing_package_target_is_retried_once_the_rebuild_finds_it( + tmp_path: Path, monkeypatch +) -> None: + # The dependency is hoisted to the run root, so the rebuild produces a different list and + # the loop has something new to try. + project = tmp_path / "smart-contracts" + project.mkdir() + (tmp_path / "node_modules" / "@vault" / "core").mkdir(parents=True) + (project / "remappings.txt").write_text("@vault/=node_modules/@vault/core/\n") + packages = [f"@vault/={project / 'node_modules/@vault/core'}/"] + manager = CompilationWorkaroundManager(project_root=tmp_path, build_config_dir=project) + logged: list[tuple[str, str]] = [] + monkeypatch.setattr(manager, "log", lambda msg, level="INFO": logged.append((msg, level))) + contracts = [ContractHandle(contract_name="Widget", source_file="src/Widget.sol")] + + success, updated, fake_run = _run_loop_with_packages( + manager, + monkeypatch, + tmp_path, + [_source_not_found(f"{project / 'node_modules/@vault/core/IVault.sol'}")], + contracts, + packages, + ) + + assert any("package_target_missing" in msg for msg, _ in logged) + # The retry compiled, so the run has no unresolved imports left to report. + assert manager.last_import_diagnostics == [] + assert success is True + assert fake_run.calls == 2 + assert updated["packages"] == [f"@vault/={tmp_path / 'node_modules/@vault/core'}/"] + + +def test_import_diagnostics_do_not_survive_into_an_unrelated_failure( + tmp_path: Path, monkeypatch +) -> None: + # The import problem IS fixed by the rebuild and the run then dies of something else. The + # classification of the first output must not be reported for that terminal failure — + # callers paste it into the compilation error, where it would blame a missing dependency. + project = tmp_path / "smart-contracts" + project.mkdir() + (tmp_path / "node_modules" / "@vault" / "core").mkdir(parents=True) + (project / "remappings.txt").write_text("@vault/=node_modules/@vault/core/\n") + packages = [f"@vault/={project / 'node_modules/@vault/core'}/"] + manager = CompilationWorkaroundManager(project_root=tmp_path, build_config_dir=project) + logged: list[tuple[str, str]] = [] + monkeypatch.setattr(manager, "log", lambda msg, level="INFO": logged.append((msg, level))) + contracts = [ContractHandle(contract_name="Widget", source_file="src/Widget.sol")] + unrelated = "Error: something entirely different, no workaround applies\n" + + success, _, _ = _run_loop_with_packages( + manager, + monkeypatch, + tmp_path, + [_source_not_found(f"{project / 'node_modules/@vault/core/IVault.sol'}")] + [unrelated] * 12, + contracts, + packages, + ) + + assert success is False + assert manager.last_import_diagnostics == [] + errors = [msg for msg, level in logged if level == "ERROR"] + assert errors and not any("package_target_missing" in msg for msg in errors) + + +def test_unparseable_source_not_found_leaves_the_loop_unchanged( + tmp_path: Path, monkeypatch +) -> None: + # A garbled diagnostic still trips the detector; classification simply finds nothing and the + # workaround runs exactly as it did before. + garbled = 'ParserError: Source " not found: File not found.\n' + manager = CompilationWorkaroundManager(project_root=tmp_path) + logged: list[tuple[str, str]] = [] + monkeypatch.setattr(manager, "log", lambda msg, level="INFO": logged.append((msg, level))) + contracts = [ContractHandle(contract_name="Widget", source_file="src/Widget.sol")] + + success, _, fake_run = _run_loop_with_packages( + manager, monkeypatch, tmp_path, [garbled] * 10, contracts, [] + ) + + assert success is False + assert manager.last_import_diagnostics == [] + # Empty project, so the rebuild reproduces the empty list and the loop gives up after the + # one certoraRun — with no diagnosis appended, exactly as before the classifier existed. + assert fake_run.calls == 1 + errors = [msg for msg, level in logged if level == "ERROR"] + assert errors and errors[-1].endswith("giving up") + + +# A contract that inherits functions it never implements. solc hard-wraps the +# diagnostic, so the sentence is split across a newline here as it is in a real +# run's output; the quoted contract name and the source location survive it. +UNIMPLEMENTED_CONTRACT_OUTPUT = ( + "Compiling certora/harnesses/TokenInstance1.sol...\n" + "solc8.35 had an error:\n" + 'TypeError: Contract "TokenInstance1" should be marked as \n' + "abstract.\n" + " --> certora/harnesses/TokenInstance1.sol:7:1:\n" + " |\n" + "7 | contract TokenInstance1 is TokenBase {\n" + " | ^ (Relevant source part starts here and spans across multiple lines).\n" + "Note: Missing implementation: \n" + " --> src/interfaces/IToken.sol:543:5:\n" + " |\n" + "543 | function transfer(address to, uint256 amount) external returns (bool);\n" +) + + +def test_detects_unimplemented_contract(manager: CompilationWorkaroundManager) -> None: + assert manager._detect_unimplemented_contract(UNIMPLEMENTED_CONTRACT_OUTPUT) == ( + "TokenInstance1", + "certora/harnesses/TokenInstance1.sol", + ) + + +def test_detects_unimplemented_contract_without_source_location( + manager: CompilationWorkaroundManager, +) -> None: + output = 'TypeError: Contract "TokenInstance1" should be marked as abstract.\n' + assert manager._detect_unimplemented_contract(output) == ("TokenInstance1", None) + + +def test_ignores_unrelated_unimplemented_contract(manager: CompilationWorkaroundManager) -> None: + assert manager._detect_unimplemented_contract(PERSISTENT_STACK_TOO_DEEP_OUTPUT) is None + + +def test_unimplemented_contract_is_terminal(manager, monkeypatch, tmp_path) -> None: + # No conf change can complete an incomplete contract: the loop must give up + # after the single certoraRun that reported it, rather than letting the + # catch-all workaround spend more compilations on the same error. + contracts = [ContractHandle(contract_name="Foo", source_file="contracts/Foo.sol")] + fake_run = _SequencedRun([UNIMPLEMENTED_CONTRACT_OUTPUT] * 10) + monkeypatch.setattr( + "certora_autosetup.utils.compilation_workarounds.subprocess.run", fake_run + ) + with pytest.raises(UnimplementedContractError) as excinfo: + manager.run_compilation_with_workarounds( + cmd=["certoraRun", "test.conf"], + config_file=tmp_path / "test.conf", + compilation_config={"files": ["contracts/Foo.sol:Foo"]}, + contracts=contracts, + updated_config_dict={}, + ) + assert "TokenInstance1" in str(excinfo.value) + assert "certora/harnesses/TokenInstance1.sol" in str(excinfo.value) + assert fake_run.calls == 1 + + +# --- the legacy stack-too-deep ladder: optimizer, then autofinder, then via-ir ---------- + +AUTOFINDER_STACK_TOO_DEEP_OUTPUT = ( + "Compiling contracts/Foo.sol to expose internal function information and local variables...\n" + "Encountered an exception generating autofinder contracts/Foo.sol (solc8.21 had an error:\n" + "CompilerError: Stack too deep. Try compiling with `--via-ir` (cli) or the equivalent\n" + "`viaIR: true` (standard JSON) while enabling the optimizer.\n" +) + + +def test_legacy_stack_too_deep_enables_the_optimizer_before_via_ir(manager, monkeypatch, tmp_path) -> None: + # solc's advice on this error names both the pipeline and the optimizer. The optimizer + # alone keeps legacy codegen, so it is what the first pass must try. + contracts = [ContractHandle(contract_name="Foo", source_file="contracts/Foo.sol")] + success, updated, config, fake_run = _run_loop( + manager, monkeypatch, tmp_path, [PERSISTENT_STACK_TOO_DEEP_OUTPUT], contracts + ) + assert success is True + assert fake_run.calls == 2 + assert updated["solc_optimize"] == "200" + assert "solc_via_ir" not in updated and "solc_via_ir_map" not in updated + + +def test_via_ir_follows_when_the_optimizer_did_not_clear_it(manager, monkeypatch, tmp_path) -> None: + # Still failing with the optimizer on: via-ir is the next rung, and it stays scoped to + # the contract that failed. + contracts = [ + ContractHandle(contract_name="Foo", source_file="contracts/Foo.sol"), + ContractHandle(contract_name="Bar", source_file="contracts/Bar.sol"), + ] + success, updated, _, fake_run = _run_loop( + manager, + monkeypatch, + tmp_path, + [PERSISTENT_STACK_TOO_DEEP_OUTPUT, PERSISTENT_STACK_TOO_DEEP_OUTPUT], + contracts, + ) + assert success is True + assert fake_run.calls == 3 + assert updated["solc_optimize"] == "200" + assert updated["solc_via_ir_map"] == {"Foo": True, "Bar": False} + + +def test_autofinder_stack_too_deep_relaxes_the_assertion_before_via_ir(manager, monkeypatch, tmp_path) -> None: + # The contracts compile; only the instrumented copies are over the limit. Accepting the + # finder fallback for those files keeps legacy codegen for every contract. + contracts = [ContractHandle(contract_name="Foo", source_file="contracts/Foo.sol")] + success, updated, config, fake_run = _run_loop( + manager, + monkeypatch, + tmp_path, + [AUTOFINDER_STACK_TOO_DEEP_OUTPUT, AUTOFINDER_STACK_TOO_DEEP_OUTPUT], + contracts, + extra_config={"assert_autofinder_success": True}, + ) + assert success is True + assert fake_run.calls == 3 + assert config["assert_autofinder_success"] is False + assert "solc_via_ir" not in updated and "solc_via_ir_map" not in updated + + +def test_via_ir_goes_scene_wide_past_the_threshold(manager, monkeypatch, tmp_path) -> None: + # Naming contracts one at a time costs a compile each; past the threshold the rest of + # the scene is switched in one step. + names = [f"C{i}" for i in range(VIA_IR_SCENE_THRESHOLD + 4)] + contracts = [ContractHandle(contract_name=n, source_file=f"contracts/{n}.sol") for n in names] + outputs = [ + f"Compiling contracts/{n}.sol...\nsolc8.21 had an error:\nCompilerError: Stack too deep.\n" + for n in names[:VIA_IR_SCENE_THRESHOLD] + ] + # One extra leading failure for the optimizer rung to consume. + success, updated, _, _ = _run_loop(manager, monkeypatch, tmp_path, [outputs[0]] + outputs, contracts) + + assert success is True + # Uniform True map collapses to the scalar on exit — every contract is on via-ir. + assert updated["solc_via_ir"] is True + assert "solc_via_ir_map" not in updated + + +def test_declared_via_ir_skips_the_per_contract_walk(manager_declaring_via_ir, monkeypatch, tmp_path) -> None: + # The project's build config already says where this ends; walking there one contract + # per compile only costs compiles. The declared value is still not inherited — the + # optimizer value emitted is ours. + contracts = [ + ContractHandle(contract_name="Foo", source_file="contracts/Foo.sol"), + ContractHandle(contract_name="Bar", source_file="contracts/Bar.sol"), + ] + success, updated, _, fake_run = _run_loop( + manager_declaring_via_ir, + monkeypatch, + tmp_path, + [PERSISTENT_STACK_TOO_DEEP_OUTPUT, PERSISTENT_STACK_TOO_DEEP_OUTPUT], + contracts, + ) + assert success is True + assert updated["solc_via_ir"] is True + assert "solc_via_ir_map" not in updated + assert updated["solc_optimize"] == "200" diff --git a/tests/test_config_edit.py b/tests/test_config_edit.py index ae6637eb..3a7919c4 100644 --- a/tests/test_config_edit.py +++ b/tests/test_config_edit.py @@ -18,6 +18,8 @@ from composer.spec.source.prover import ( OVERLAY_OWNED_KEYS, ProverStateExtra, prover_config_overlay, ) +from composer.spec.system_model import SolidityIdentifier +from composer.spec.source.prover import ProverStateExtra from graphcore.testing import Scenario, tool_call_raw, ToolCallDict @@ -48,7 +50,7 @@ def _add_file( compiler_settings: CompilerSettings | None = None, ) -> dict: return AddFile( - type="add_file", file_path=path, contract_name=contract_name, + type="add_file", file_path=path, contract_name=SolidityIdentifier(contract_name) if contract_name is not None else None, compiler_settings=compiler_settings, ).model_dump() @@ -88,7 +90,7 @@ def _scenario( if extra is not None: config.update(extra) return Scenario(ConfigTestState, TOOL).init( - config=config, rule_skips={}, + config=config, rule_skips={}, reminders_channel=[] ) diff --git a/tests/test_contract_utils_nested_project.py b/tests/test_contract_utils_nested_project.py index 4fde4279..316b6340 100644 --- a/tests/test_contract_utils_nested_project.py +++ b/tests/test_contract_utils_nested_project.py @@ -10,7 +10,7 @@ import pytest import certora_autosetup.utils.contract_utils as cu -from certora_autosetup.utils.contract_utils import auto_detect_contracts, resolve_contract_handles +from certora_autosetup.utils.contract_utils import auto_detect_contracts, resolve_contract_handles, with_contract_handle from certora_autosetup.utils.types import ContractHandle @@ -83,3 +83,33 @@ def build_source_path_to_contracts_map(self): assert resolved[0].contract_name == "TheRealName" # The returned path stays in the caller's frame of reference. assert resolved[0].source_file == "sub/src/Widget.sol" + + +def test_main_contract_added_when_auto_detection_missed_it() -> None: + # Auto-detection skips anything under a dependency directory, which is where per-address + # verification bundles keep the deployed code. Without this the main contract never reaches + # the compilation conf and the run dies at "not among the compiled contracts". + detected = [ContractHandle("Widget", "src/Widget.sol")] + main = ContractHandle("Gadget", "src/Gadget_1234/dependencies/pkg-1.0.0/src/Gadget.sol") + + handles = with_contract_handle(detected, main) + + assert main in handles + assert detected[0] in handles + + +def test_main_contract_displaces_a_same_named_handle_from_another_file() -> None: + # Dedup keeps the shortest path per contract name, which can pick a different file than the + # one the caller named. The caller's file wins, and only one handle carries the name. + detected = [ContractHandle("Widget", "src/Widget.sol")] + main = ContractHandle("Widget", "src/vendor/bundle/src/Widget.sol") + + handles = with_contract_handle(detected, main) + + assert handles == [main] + + +def test_already_detected_main_contract_is_left_alone() -> None: + detected = [ContractHandle("Widget", "src/Widget.sol"), ContractHandle("Gear", "src/Gear.sol")] + + assert with_contract_handle(detected, detected[0]) == detected diff --git a/tests/test_cvl_skips.py b/tests/test_cvl_skips.py index a3ce0fb4..974d8f18 100644 --- a/tests/test_cvl_skips.py +++ b/tests/test_cvl_skips.py @@ -1,8 +1,8 @@ """ Tests for CVL generation skip/completion machinery wired through a ReAct graph. -Uses a minimal test tool to exercise the _merge_skips reducer and -check_completion / _compute_digest validation logic end-to-end. +Uses a minimal test tool to exercise the merge_skips reducer and +check_completion / spec_digest validation logic end-to-end. """ import pytest @@ -16,10 +16,10 @@ from langgraph.graph import MessagesState +from composer.authoring.state import SkippedProperty, check_completion +from composer.spec.types import PropertyTitle from composer.spec.cvl_generation import ( CVLGenerationExtra, - SkippedProperty, - check_completion, property_tools, FEEDBACK_VALIDATION_KEY, FeedbackServices, @@ -156,7 +156,7 @@ def scenario( ): services = FeedbackServices( feedback_thunk=feedback_impl, - titles=[f"p{i}" for i in range(num_props)], + titles=[PropertyTitle(f"p{i}") for i in range(num_props)], ) return Scenario(CVLTestState, *property_tools(services), *_STATIC_TOOLS).init( curr_spec=curr_spec, @@ -168,7 +168,7 @@ def scenario( # ========================================================================= -# _merge_skips reducer via graph +# merge_skips reducer via graph # ========================================================================= diff --git a/tests/test_document_discovery.py b/tests/test_document_discovery.py new file mode 100644 index 00000000..a3132ffb --- /dev/null +++ b/tests/test_document_discovery.py @@ -0,0 +1,86 @@ +"""Directory sweeping for ``--extra-context``. + +A ``--extra-context`` entry is either a document or a directory to sweep. The resolved +list feeds the prompt and the bug-analysis cache key, and ``cache-autoprove inputs`` +re-runs the same resolver to rebuild that key, so it must be deterministic and +reproducible from the CLI arguments alone. The sweep is deliberately flat. +""" + +import pathlib + +import pytest + +from composer.input.files import discover_documents, resolve_document_paths + + +def _tree(root: pathlib.Path, *relative: str) -> None: + for rel in relative: + p = root / rel + p.parent.mkdir(parents=True, exist_ok=True) + p.write_text(f"contents of {rel}\n") + + +def test_sweep_is_flat_and_sorted_by_name(tmp_path: pathlib.Path) -> None: + _tree(tmp_path, "b.md", "a.md", "nested/deep/z.txt", "nested/a.md") + # Not recursive: nested/ is invisible to the sweep. + assert [p.name for p in discover_documents(tmp_path)] == ["a.md", "b.md"] + + +def test_filters_by_suffix_case_insensitively(tmp_path: pathlib.Path) -> None: + _tree(tmp_path, "notes.MD", "scope.Pdf", "readme.rst", "data.json", "Vault.sol") + got = {p.name for p in discover_documents(tmp_path)} + # Source and config are deliberately excluded from a sweep, even though the + # uploader would happily read them if named explicitly. + assert got == {"notes.MD", "scope.Pdf", "readme.rst"} + + +def test_skips_hidden_files(tmp_path: pathlib.Path) -> None: + _tree(tmp_path, "keep.md", ".hidden.md") + assert [p.name for p in discover_documents(tmp_path)] == ["keep.md"] + + +def test_empty_sweep_returns_empty_list(tmp_path: pathlib.Path) -> None: + _tree(tmp_path, "Vault.sol", "nested/a.md") + assert discover_documents(tmp_path) == [] + + +def test_non_directory_is_an_error(tmp_path: pathlib.Path) -> None: + _tree(tmp_path, "a.md") + with pytest.raises(ValueError, match="not a directory"): + discover_documents(tmp_path / "a.md") + with pytest.raises(ValueError, match="not a directory"): + discover_documents(tmp_path / "nope") + + +# --- resolve_document_paths ------------------------------------------------------------ + +def test_entries_keep_their_order_with_sweeps_spliced_in_place(tmp_path: pathlib.Path) -> None: + _tree(tmp_path, "z.md", "a.md", "swept/m.md", "swept/b.md") + got = resolve_document_paths([ + str(tmp_path / "z.md"), str(tmp_path / "swept"), str(tmp_path / "a.md"), + ]) + assert [p.name for p in got] == ["z.md", "b.md", "m.md", "a.md"] + + +def test_no_entries_is_empty() -> None: + assert resolve_document_paths(None) == [] + assert resolve_document_paths([]) == [] + + +def test_named_file_is_not_suffix_filtered(tmp_path: pathlib.Path) -> None: + # A named file bypasses DOCUMENT_SUFFIXES — that filter is a sweep policy only. + _tree(tmp_path, "Vault.sol") + assert [p.name for p in resolve_document_paths([str(tmp_path / "Vault.sol")])] == [ + "Vault.sol" + ] + + +def test_missing_entry_is_an_error(tmp_path: pathlib.Path) -> None: + with pytest.raises(ValueError, match="no such file or directory"): + resolve_document_paths([str(tmp_path / "nope.md")]) + + +def test_directory_with_no_documents_is_an_error(tmp_path: pathlib.Path) -> None: + _tree(tmp_path, "empty/Vault.sol") + with pytest.raises(ValueError, match="no supported documents"): + resolve_document_paths([str(tmp_path / "empty")]) diff --git a/tests/test_edit_cvl.py b/tests/test_edit_cvl.py new file mode 100644 index 00000000..7c0961eb --- /dev/null +++ b/tests/test_edit_cvl.py @@ -0,0 +1,74 @@ +"""Tests for ``edit_cvl``'s parallel-call guard, wired through a mocked ReAct +graph (``graphcore.testing.Scenario``). + +An AI turn carrying more than one ``edit_cvl`` call must be refused: parallel +edits against the same buffer would race through the state reducer. The guard +is per-tool, not per-turn — a single ``edit_cvl`` alongside a *different* tool +call is allowed (unlike ``verify_spec``'s solo-turn rule). + +Both tests stay on paths that return before ``maybe_update_cvl``, so the real +CVL parser (Typechecker jar) is never launched — this is a fast unit test. +""" +import pytest + +from langgraph.graph import MessagesState + +from composer.cvl.tools import WithCurrSpec, edit_cvl, get_cvl + +from graphcore.testing import Scenario, tool_call_raw + +pytestmark = pytest.mark.asyncio + + +class EditTestState(MessagesState, WithCurrSpec): + pass + + +EDIT_TOOL = edit_cvl(EditTestState) +GET_TOOL = get_cvl(EditTestState) + +# The exact refusal edit_cvl returns for a parallel-edit turn. +_REFUSAL = "`edit_cvl` tool cannot be called in parallel within the same turn." + +_SPEC = """\ +rule foo { + assert true; +} +""" + + +def _edit(old: str, new: str): + return tool_call_raw("edit_cvl", old_string=old, new_string=new) + + +def _scenario(): + return Scenario(EditTestState, EDIT_TOOL, GET_TOOL).init(curr_spec=_SPEC) + + +async def test_parallel_edits_both_refused(): + st = await _scenario().turn( + _edit("assert true", "assert false"), + _edit("rule foo", "rule bar"), + ).run() + responses = [p["resp"] for p in Scenario.get_last_tool_result(st)["edit_cvl"]] + assert len(responses) == 2 + assert all(r == _REFUSAL for r in responses) + # Neither edit landed: the buffer is untouched. + assert st["curr_spec"] == _SPEC + + +async def test_single_edit_alongside_other_tool_not_refused(): + """One edit_cvl next to a different tool is not a parallel-edit turn. The + edit targets a span absent from the buffer, so it fails in replace_unique + — deterministically past the guard, without reaching the CVL parser.""" + st = await _scenario().turn( + _edit("does not appear in the buffer", "irrelevant"), + tool_call_raw("get_cvl"), + ).run() + results = Scenario.get_last_tool_result(st) + (edit_pair,) = results["edit_cvl"] + assert edit_pair["resp"] != _REFUSAL + assert "`old_string` was not found" in edit_pair["resp"] + # The sibling call was serviced normally (responses are harness-stripped). + (get_pair,) = results["get_cvl"] + assert get_pair["resp"] == _SPEC.strip() diff --git a/tests/test_extra_context_cache_key.py b/tests/test_extra_context_cache_key.py new file mode 100644 index 00000000..355d2526 --- /dev/null +++ b/tests/test_extra_context_cache_key.py @@ -0,0 +1,75 @@ +"""The ``--extra-context`` documents must parameterize the bug-analysis cache key. + +Two runs differing only in extra context must not share cached properties. The flag takes +a list, so adding a document, dropping one, or reordering them all change the prompt and +must all change the key. A run without extra context keeps the bare ``bug_analysis`` key, +so existing caches stay addressable. +""" + +from composer.pipeline.run_tags import AutoProveCacheTags +from composer.spec.context import CacheKey +from composer.spec.prop_inference import BUG_ANALYSIS_KEY +from composer.spec.util import combine_digests + + +def _key(k: CacheKey) -> str: + """``CacheKey`` is an opaque wrapper with no ``__eq__`` — compare the string.""" + return str(k) + + +# --- combine_digests ------------------------------------------------------------------ + +def test_combine_digests_is_fixed_width() -> None: + # Folded, not concatenated — the key does not grow with the document count. + one, two = combine_digests(["xc1"]), combine_digests(["xc1", "xc2"]) + assert one is not None and two is not None + assert len(one) == len(two) == len(combine_digests(["a"] * 50) or "") + + +# --- key layout ----------------------------------------------------------------------- + +def test_no_extra_inputs_keeps_the_historical_key() -> None: + assert _key(BUG_ANALYSIS_KEY(None, with_refinement=False)) == "bug_analysis" + assert _key(BUG_ANALYSIS_KEY(None, with_refinement=True)) == "bug_analysis|refine" + assert _key(BUG_ANALYSIS_KEY("tm1", with_refinement=False)) == "bug_analysis-tm-tm1" + # ...and an empty document list folds to None, which must not perturb them. + assert combine_digests([]) is None + assert _key(BUG_ANALYSIS_KEY( + "tm1", with_refinement=True, extra_context_digest=combine_digests([]) + )) == "bug_analysis|refine-tm-tm1" + + +def test_extra_context_composes_with_the_other_inputs() -> None: + xc = combine_digests(["xc1"]) + assert _key(BUG_ANALYSIS_KEY( + None, with_refinement=False, extra_context_digest=xc, + )) == f"bug_analysis-xc-{xc}" + assert _key(BUG_ANALYSIS_KEY( + "tm1", with_refinement=True, extra_context_digest=xc, + )) == f"bug_analysis|refine-tm-tm1-xc-{xc}" + + +def test_document_list_is_order_and_membership_sensitive() -> None: + keys = [ + _key(BUG_ANALYSIS_KEY(None, False, combine_digests(docs))) + for docs in ([], ["xc1"], ["xc2"], ["xc1", "xc2"], ["xc2", "xc1"], ["xc1", "xc1"]) + ] + assert len(set(keys)) == len(keys), keys + + +# --- run tags ------------------------------------------------------------------------- + +def test_run_tags_carry_the_digests_in_order() -> None: + tags = AutoProveCacheTags( + cache_root=["ns"], contract_name="C", memory_ns=None, + threat_model_digest="tm1", extra_context_digests=["xc1", "xc2"], interactive=False, + ) + restored = AutoProveCacheTags.model_validate(tags.model_dump()) + assert restored.extra_context_digests == ["xc1", "xc2"] + + +def test_old_run_tags_default_to_no_extra_context() -> None: + legacy = {"cache_root": ["ns"], "contract_name": "C", "memory_ns": None} + tags = AutoProveCacheTags.model_validate(legacy) + assert tags.extra_context_digests == [] + assert combine_digests(tags.extra_context_digests) is None diff --git a/tests/test_foundry_entry_args.py b/tests/test_foundry_entry_args.py index fcb9fa21..1e4cec5b 100644 --- a/tests/test_foundry_entry_args.py +++ b/tests/test_foundry_entry_args.py @@ -12,6 +12,33 @@ from composer.foundry.entry import _build_parser +def test_foundry_parser_accepts_repeated_extra_context() -> None: + """``--extra-context`` is repeatable, and must not swallow the positionals — this + parser has three of them, one optional, which is why it is ``action="append"`` + rather than ``nargs="+"``.""" + args = _build_parser().parse_args([ + "proj", "src/C.sol:C", "doc.md", + "--extra-context", "a.md", "--extra-context", "b.md", + ]) + assert args.extra_context == ["a.md", "b.md"] + + # Flag ahead of the positionals must still leave all three intact. + args = _build_parser().parse_args([ + "--extra-context", "a.md", "proj", "src/C.sol:C", "doc.md", + ]) + assert args.extra_context == ["a.md"] + assert (args.project_root, args.main_contract, args.system_doc) == ( + "proj", "src/C.sol:C", "doc.md", + ) + + +def test_foundry_parser_defaults_extra_context_and_threat_model() -> None: + args = _build_parser().parse_args(["proj", "src/C.sol:C"]) + assert args.extra_context is None + # Foundry does not expose --threat-model; it stays a set_defaults stub. + assert args.threat_model is None + + def test_foundry_parser_provides_model_tier_args() -> None: args = _build_parser().parse_args(["proj", "src/C.sol:C", "doc.md"]) for attr in ( diff --git a/tests/test_foundry_runner.py b/tests/test_foundry_runner.py index 8394a4e5..8fafbe8e 100644 --- a/tests/test_foundry_runner.py +++ b/tests/test_foundry_runner.py @@ -34,11 +34,11 @@ def test_parse_forge_json_parses_a_minimal_report() -> None: assert [(r.name, r.status) for r in results] == [("test_Foo", "Success")] -def _min_state(curr_test: str) -> dict: +def _min_state(curr_spec: str) -> dict: """The minimal FoundryGenerationState the runner's build-failure branch reads.""" return { "messages": [], - "curr_test": curr_test, + "curr_spec": curr_spec, "skipped": [], "property_tests": [], "validations": {}, @@ -46,6 +46,7 @@ def _min_state(curr_test: str) -> dict: "expected_failures": {}, "last_test_names": ["stale_name"], "failed": None, + "budget_curtailed": False } diff --git a/tests/test_fuzzed_templates.py b/tests/test_fuzzed_templates.py index 479da3be..ddb9280d 100644 --- a/tests/test_fuzzed_templates.py +++ b/tests/test_fuzzed_templates.py @@ -17,15 +17,22 @@ HarnessedApplication, _context_marker_attr, BaseApplication ) + from composer.spec.solana.model import ( SolanaApplication, SolanaComponentInstance, SolanaProgramInstance ) + +from composer.spec.soroban.model import ( + SorobanApplication, SorobanComponentInstance, SorobanContractInstance +) + from composer.templates.loader import _patch_environment_filters from composer.spec.service_host import Sort # defined here? huh? import hypothesis.strategies._internal.core as hcore import os + REPO_ROOT = pathlib.Path(__file__).parent.parent TEMPLATES_DIR = REPO_ROOT / "composer" / "templates" @@ -366,6 +373,28 @@ def solana_component_resolver(t: type) -> st.SearchStrategy[SolanaComponentInsta ) ) +def soroban_contract_resolver(t: type) -> st.SearchStrategy[SorobanContractInstance]: + return st.from_type(SorobanApplication).filter( + lambda a: len(a.contracts) > 0 + ).flatmap(lambda app: \ + st.builds( + SorobanContractInstance, + ind=st.integers(min_value=0, max_value=len(app.contracts) - 1), + app=st.just(app) + ) + ) + +def soroban_component_resolver(t: type) -> st.SearchStrategy[SorobanComponentInstance]: + return st.from_type(SorobanContractInstance).filter( + lambda c: len(c.contract.components) > 0 + ).flatmap(lambda contract: \ + st.builds( + SorobanComponentInstance, + ind=st.integers(min_value=0, max_value=len(contract.contract.components) - 1), + _contract=st.just(contract) + ) + ) + def sort_to_application( sort: Sort ) -> st.SearchStrategy[AnyApplication]: @@ -383,14 +412,15 @@ def sort_to_application( st.register_type_strategy(SolanaComponentInstance, solana_component_resolver) -#: How to draw a marked param dict's ``context``, per concrete unit type the ecosystems declare -#: (see ``_build_template_context``). EVM's is sort-coherent: its prompts branch on ``sort`` and -#: read subtype-specific fields off the app, so the unit must come from an app of the matching -#: family. Solana's templates have no ``sort`` branch (no greenfield/update split), so its unit is -#: drawn independently — the registered resolver already keeps the indices in bounds. +st.register_type_strategy(SorobanContractInstance, soroban_contract_resolver) + +st.register_type_strategy(SorobanComponentInstance, soroban_component_resolver) + +#: How to draw a marked param dict's ``context``, per concrete unit type. _COHERENT_UNITS: dict[type, Callable[[Sort], st.SearchStrategy[Any]]] = { ContractComponentInstance: lambda sort: contract_to_component(app_to_contract(sort_to_application(sort))), SolanaComponentInstance: lambda _sort: st.from_type(SolanaComponentInstance), + SorobanComponentInstance: lambda _sort: st.from_type(SorobanComponentInstance), } settings.register_profile("quick", settings( diff --git a/tests/test_graph_retry.py b/tests/test_graph_retry.py new file mode 100644 index 00000000..c63eb6ff --- /dev/null +++ b/tests/test_graph_retry.py @@ -0,0 +1,561 @@ +"""Tests for the retry machinery in ``composer.io.context`` — the ambient +(run-wide) retry floor, per-run overrides, and the ``FreshRetryPolicy`` +escalation, all first-class in ``run_graph``. + +The graphs are static pregel loops (plan → author) over in-memory +checkpointers; nodes consult a scripted ``FlakyLLM`` that raises on the turns +the script says to. No Postgres, no real backoff sleeps (policies get a +recording backoff), no LLM. + +What the scenarios pin down: + +* a retryable failure resumes from the LAST CHECKPOINT — completed nodes do + not re-run, the failed node does, and the backoff is consulted with the + failed attempt's index; +* a non-retryable failure propagates immediately, no backoff consulted; +* the ambient floor (``install_retry_policy``) applies when no per-run policy + is passed — and to NESTED sub-graph runs, which each track their own + checkpoints via their own sink wrapper; +* a per-run policy that is a ``RetryPolicy`` overrides the ambient floor; +* the state-backoff ladder: a sub-agent that EXHAUSTS its floor bubbles into + the parent's floor retry, which re-enters the spawning node — respawning + the sub-agent on a fresh thread id with pristine state, while the exhausted + spawn's thread is abandoned mid-flight; +* a plain ``FreshRetryPolicy`` (deliberately NOT a ``RetryPolicy``) rides the + ambient floor for transient failures while escalating wedged ones by + rebuilding the input on a fresh thread — attempt 1's state does not leak; +* exhaustion, both loops: a retryable failure on the final attempt propagates + as-is — no parting backoff, no pointless final ``rebuild_input``. +""" +import operator +import uuid +from contextlib import contextmanager +from typing import Annotated, Any, TypedDict, override + +import pytest + +from langgraph.checkpoint.memory import InMemorySaver +from langgraph.graph import StateGraph, START, END +from langgraph.graph.state import CompiledStateGraph + +from composer.io.context import ( + Backoff, + DefaultRetryPolicy, + FreshRetryPolicy, + RetryPolicy, + install_retry_policy, + run_to_completion, + with_handler, +) +from composer.io.event_handler import NullEventHandler + +pytestmark = pytest.mark.asyncio + + +class RetryState(TypedDict): + log: Annotated[list[str], operator.add] + + +class FakeOverloadedError(Exception): + """Stands in for the transiently-retryable class (529/rate limit).""" + + +class FakeCorruptedError(Exception): + """Stands in for a wedged-thread failure that needs a fresh start.""" + + +class FakeFatalError(Exception): + """Retryable by no policy.""" + + +def _retry_on(*types: type[Exception]): + return lambda e: isinstance(e, types) + + +class FlakyLLM: + """Scripted fake: each ``ainvoke`` consumes the next script entry — + a str to return, or an exception instance to raise.""" + + def __init__(self, script: list[str | Exception]): + self.script = list(script) + self.calls = 0 + + async def ainvoke(self, prompt: str) -> str: + i = self.calls + self.calls += 1 + assert i < len(self.script), f"FlakyLLM script exhausted at call {i} ({prompt!r})" + item = self.script[i] + if isinstance(item, Exception): + raise item + return item + + +class _RecordingIOHandler: + """IOHandler that records log_start descriptions (the retry loops label + attempts through them) and refuses HITL (none expected here).""" + + def __init__(self) -> None: + self.descriptions: list[str] = [] + + async def log_checkpoint_id(self, *, path: list[str], checkpoint_id: str) -> None: + pass + + async def log_state_update(self, path: list[str], st: dict) -> None: + pass + + async def log_start(self, *, path: list[str], description: str, tool_id: str | None) -> None: + self.descriptions.append(description) + + async def log_end(self, path: list[str]) -> None: + pass + + async def human_interaction(self, ty: Any, debug_thunk: Any) -> str: + raise AssertionError("no HITL interaction expected in retry tests") + + +def _recording_backoff(record: list[int]) -> Backoff: + async def _backoff(i: int) -> None: + record.append(i) + return _backoff + + +@contextmanager +def _ambient_policy(policy: RetryPolicy): + tok = install_retry_policy(policy) + try: + yield + finally: + tok.var.reset(tok) + + +def _build_graph( + llm: FlakyLLM, node_runs: list[str] +) -> CompiledStateGraph[RetryState, None, RetryState, RetryState]: + async def plan(state: RetryState) -> dict: + node_runs.append("plan") + return {"log": [f"plan: {await llm.ainvoke('plan')}"]} + + async def author(state: RetryState) -> dict: + node_runs.append("author") + return {"log": [f"author: {await llm.ainvoke('author')}"]} + + builder = StateGraph(RetryState) + builder.add_node("plan", plan) + builder.add_node("author", author) + builder.add_edge(START, "plan") + builder.add_edge("plan", "author") + builder.add_edge("author", END) + return builder.compile(checkpointer=InMemorySaver()) + + +async def _run( + graph: CompiledStateGraph[RetryState, None, RetryState, RetryState], + handler: _RecordingIOHandler, + *, + thread_id: str, + retry: "RetryPolicy | FreshRetryPolicy[Any, Any] | None" = None, +) -> RetryState: + async with with_handler(handler, NullEventHandler()): + return await run_to_completion( + graph, + {"log": [""]}, + thread_id=thread_id, + context=None, + recursion_limit=25, + description="retry test", + retry=retry, + ) + + +# ========================================================================= +# Floor policy: resume-from-checkpoint, propagation, exhaustion +# ========================================================================= + + +async def test_retryable_failures_resume_from_last_checkpoint(): + node_runs: list[str] = [] + llm = FlakyLLM([ + "planned", + FakeOverloadedError("529 attempt 1"), + FakeOverloadedError("529 attempt 2"), + "authored", + ]) + backoffs: list[int] = [] + handler = _RecordingIOHandler() + policy = DefaultRetryPolicy( + _retry_on(FakeOverloadedError), backoff=_recording_backoff(backoffs), max_retries=3, + ) + + result = await _run(_build_graph(llm, node_runs), handler, thread_id="retry-resume", retry=policy) + + assert result["log"] == ["", "plan: planned", "author: authored"] + # Resume, not restart: plan's checkpoint survived the failures, so plan ran + # exactly once while the failing author task re-ran on each attempt. + assert node_runs == ["plan", "author", "author", "author"] + assert llm.calls == 4 + # Backoff consulted once per failed attempt, with that attempt's index. + assert backoffs == [0, 1] + # Attempts are labeled for the handler. + assert handler.descriptions == [ + "retry test", "retry test (Retry 1)", "retry test (Retry 2)", + ] + + +async def test_non_retryable_failure_propagates_without_backoff(): + node_runs: list[str] = [] + llm = FlakyLLM(["planned", FakeFatalError("boom")]) + backoffs: list[int] = [] + handler = _RecordingIOHandler() + policy = DefaultRetryPolicy( + _retry_on(FakeOverloadedError), backoff=_recording_backoff(backoffs), max_retries=3, + ) + + with pytest.raises(FakeFatalError): + await _run(_build_graph(llm, node_runs), handler, thread_id="retry-fatal", retry=policy) + + assert node_runs == ["plan", "author"] + assert backoffs == [] + + +async def test_retry_exhaustion_raises_last_error_without_parting_backoff(): + node_runs: list[str] = [] + llm = FlakyLLM([ + "planned", + FakeOverloadedError("529 attempt 1"), + FakeOverloadedError("529 attempt 2"), + ]) + backoffs: list[int] = [] + handler = _RecordingIOHandler() + policy = DefaultRetryPolicy( + _retry_on(FakeOverloadedError), backoff=_recording_backoff(backoffs), max_retries=2, + ) + + with pytest.raises(FakeOverloadedError, match="attempt 2"): + await _run(_build_graph(llm, node_runs), handler, thread_id="retry-exhausted", retry=policy) + + assert node_runs == ["plan", "author", "author"] + # Backoff paid between attempts only — the final failure exits immediately. + assert backoffs == [0] + + +# ========================================================================= +# Ambient floor: applies by default, overridable per-run, reaches sub-graphs +# ========================================================================= + + +async def test_ambient_floor_applies_without_explicit_policy(): + node_runs: list[str] = [] + llm = FlakyLLM(["planned", FakeOverloadedError("blip"), "authored"]) + backoffs: list[int] = [] + handler = _RecordingIOHandler() + + with _ambient_policy(DefaultRetryPolicy( + _retry_on(FakeOverloadedError), backoff=_recording_backoff(backoffs), max_retries=3, + )): + result = await _run(_build_graph(llm, node_runs), handler, thread_id="ambient-floor") + + assert result["log"] == ["", "plan: planned", "author: authored"] + assert node_runs == ["plan", "author", "author"] + assert backoffs == [0] + + +async def test_explicit_policy_overrides_the_ambient_floor(): + """A per-run RetryPolicy replaces the ambient floor entirely: the ambient + policy would have retried this failure, the override refuses to.""" + node_runs: list[str] = [] + llm = FlakyLLM(["planned", FakeOverloadedError("would be retried ambiently")]) + ambient_backoffs: list[int] = [] + override_backoffs: list[int] = [] + handler = _RecordingIOHandler() + + with _ambient_policy(DefaultRetryPolicy( + _retry_on(FakeOverloadedError), backoff=_recording_backoff(ambient_backoffs), max_retries=3, + )): + with pytest.raises(FakeOverloadedError): + await _run( + _build_graph(llm, node_runs), handler, thread_id="floor-override", + retry=DefaultRetryPolicy( + lambda e: False, backoff=_recording_backoff(override_backoffs), max_retries=3, + ), + ) + + assert node_runs == ["plan", "author"] + assert ambient_backoffs == [] + assert override_backoffs == [] + + +async def test_nested_subgraph_failures_retry_under_the_ambient_floor(): + """The capability the sink-wrapper tracking buys: a transient failure + inside a NESTED graph run retries from the nested run's own checkpoint, + without the failure ever reaching (or re-running) the parent.""" + child_runs: list[str] = [] + child_llm = FlakyLLM([FakeOverloadedError("nested blip"), "child works"]) + backoffs: list[int] = [] + handler = _RecordingIOHandler() + + async def child_work(state: RetryState) -> dict: + child_runs.append("child-work") + return {"log": [f"child: {await child_llm.ainvoke('child')}"]} + + child_builder = StateGraph(RetryState) + child_builder.add_node("child_work", child_work) + child_builder.add_edge(START, "child_work") + child_builder.add_edge("child_work", END) + child_graph = child_builder.compile(checkpointer=InMemorySaver()) + + parent_runs: list[str] = [] + + async def delegate(state: RetryState) -> dict: + parent_runs.append("delegate") + child_state = await run_to_completion( + child_graph, + {"log": [""]}, + thread_id="nested-child", + context=None, + recursion_limit=25, + description="nested child", + ) + return {"log": [f"delegate got: {child_state['log'][-1]}"]} + + parent_builder = StateGraph(RetryState) + parent_builder.add_node("delegate", delegate) + parent_builder.add_edge(START, "delegate") + parent_builder.add_edge("delegate", END) + parent_graph = parent_builder.compile(checkpointer=InMemorySaver()) + + with _ambient_policy(DefaultRetryPolicy( + _retry_on(FakeOverloadedError), backoff=_recording_backoff(backoffs), max_retries=3, + )): + async with with_handler(handler, NullEventHandler()): + result = await run_to_completion( + parent_graph, + {"log": [""]}, + thread_id="nested-parent", + context=None, + recursion_limit=25, + description="parent", + ) + + # The nested failure was retried in place: the child node re-ran, the + # parent's delegate node ran exactly once, and the result flowed through. + assert child_runs == ["child-work", "child-work"] + assert parent_runs == ["delegate"] + assert backoffs == [0] + assert result["log"] == ["", "delegate got: child: child works"] + # The retry was labeled on the nested run, not the parent. + assert "nested child (Retry 1)" in handler.descriptions + assert "parent (Retry 1)" not in handler.descriptions + + +async def test_exhausted_subagent_escalates_to_parent_and_respawns_fresh(): + """The state-backoff ladder, end to end. Spawn 1's inner floor retries + resume the sub-agent's own checkpoint (level 1); once exhausted, the + failure bubbles into the parent's floor retry (level 2), which resumes + the PARENT's checkpoint and re-enters the spawning node — respawning the + sub-agent under a fresh thread id with pristine state. Each level rolls + back strictly more state.""" + act_llm = FlakyLLM([ + FakeOverloadedError("spawn 1, act attempt 1"), + FakeOverloadedError("spawn 1, act attempt 2"), + FakeOverloadedError("spawn 2, act attempt 1"), + "acted", + ]) + child_runs: list[str] = [] + gather_views: list[tuple[str, ...]] = [] + + async def gather(state: RetryState) -> dict: + child_runs.append("gather") + gather_views.append(tuple(state["log"])) + return {"log": ["gathered"]} + + async def act(state: RetryState) -> dict: + child_runs.append("act") + return {"log": [f"act: {await act_llm.ainvoke('act')}"]} + + child_builder = StateGraph(RetryState) + child_builder.add_node("gather", gather) + child_builder.add_node("act", act) + child_builder.add_edge(START, "gather") + child_builder.add_edge("gather", "act") + child_builder.add_edge("act", END) + child_graph = child_builder.compile(checkpointer=InMemorySaver()) + + parent_runs: list[str] = [] + spawned_tids: list[str] = [] + + async def plan(state: RetryState) -> dict: + parent_runs.append("plan") + return {"log": ["planned"]} + + async def delegate(state: RetryState) -> dict: + parent_runs.append("delegate") + # Mirrors how tool bodies spawn sub-agents: a unique thread id per + # spawn, so a parent-level retry starts the sub-agent over instead of + # resuming the exhausted spawn's conversation. + tid = f"ladder-child-{uuid.uuid4().hex}" + spawned_tids.append(tid) + child_state = await run_to_completion( + child_graph, + {"log": [""]}, + thread_id=tid, + context=None, + recursion_limit=25, + description="child work", + ) + return {"log": [f"delegate got: {child_state['log'][-1]}"]} + + parent_builder = StateGraph(RetryState) + parent_builder.add_node("plan", plan) + parent_builder.add_node("delegate", delegate) + parent_builder.add_edge(START, "plan") + parent_builder.add_edge("plan", "delegate") + parent_builder.add_edge("delegate", END) + parent_graph = parent_builder.compile(checkpointer=InMemorySaver()) + + backoffs: list[int] = [] + handler = _RecordingIOHandler() + with _ambient_policy(DefaultRetryPolicy( + _retry_on(FakeOverloadedError), backoff=_recording_backoff(backoffs), max_retries=2, + )): + async with with_handler(handler, NullEventHandler()): + result = await run_to_completion( + parent_graph, + {"log": [""]}, + thread_id="ladder-parent", + context=None, + recursion_limit=25, + description="parent ladder", + ) + + # Level 1 (within a spawn): inner retries RESUME — act re-ran without + # gather re-running. Level 2 (across spawns): the respawn is FRESH — + # gather ran again. + assert child_runs == ["gather", "act", "act", "gather", "act", "act"] + # The parent resumed from its checkpoint: plan ran once, delegate re-ran. + assert parent_runs == ["plan", "delegate", "delegate"] + # Two distinct spawns... + assert len(spawned_tids) == 2 and spawned_tids[0] != spawned_tids[1] + # ...and both saw pristine state: no trace of the sibling spawn's history. + assert gather_views == [("",), ("",)] + # One backoff per failure that had a next attempt: spawn 1's inner retry, + # the parent's retry, spawn 2's inner retry — spawn 1's exhausting second + # failure paid none. + assert backoffs == [0, 0, 0] + assert result["log"] == ["", "planned", "delegate got: act: acted"] + # The exhausted spawn's thread is abandoned mid-flight; the fresh spawn's + # completed — "backing off the state". + spawn1 = (await child_graph.aget_state({"configurable": {"thread_id": spawned_tids[0]}})).values + spawn2 = (await child_graph.aget_state({"configurable": {"thread_id": spawned_tids[1]}})).values + assert spawn1["log"] == ["", "gathered"] + assert spawn2["log"] == ["", "gathered", "act: acted"] + # Each level labels its own attempts. + assert handler.descriptions == [ + "parent ladder", + "child work", "child work (Retry 1)", + "parent ladder (Retry 1)", + "child work", "child work (Retry 1)", + ] + + +# ========================================================================= +# Fresh-start escalation +# ========================================================================= + + +class _RecordingFreshPolicy(FreshRetryPolicy[Any, Any]): + """Fresh-only escalation — deliberately NOT a RetryPolicy, so the floor + (if any) comes from the ambient installation.""" + + max_fresh_retries = 2 + + def __init__(self, next_thread_id: str): + self._next_thread_id = next_thread_id + self.rebuilds: list[tuple[Any, Any]] = [] + + @override + def should_retry_fresh(self, exc: Exception) -> bool: + return isinstance(exc, FakeCorruptedError) + + @override + async def rebuild_input(self, last_state: Any, last_input: Any) -> tuple[Any, str]: + self.rebuilds.append((last_state, last_input)) + return ({"log": [""]}, self._next_thread_id) + + +async def test_fresh_start_rebuilds_input_on_a_fresh_thread(): + node_runs: list[str] = [] + llm = FlakyLLM([ + "planned v1", + FakeCorruptedError("thread is wedged"), + "planned v2", + "authored v2", + ]) + handler = _RecordingIOHandler() + policy = _RecordingFreshPolicy(next_thread_id="retry-fresh-2") + + result = await _run(_build_graph(llm, node_runs), handler, thread_id="retry-fresh-1", retry=policy) + + # The fresh attempt ran on the rebuilt input, and attempt 1's state did NOT + # leak into it: a fresh thread id is a genuinely fresh history. + assert result["log"] == ["", "plan: planned v2", "author: authored v2"] + # Restart, not resume: the whole graph re-ran on the fresh thread. + assert node_runs == ["plan", "author", "plan", "author"] + # rebuild_input saw the crashed attempt's checkpointed state (plan's work) + # and the input that attempt ran with. + assert len(policy.rebuilds) == 1 + (last_state, last_input) = policy.rebuilds[0] + assert last_state["log"] == ["", "plan: planned v1"] + assert last_input == {"log": [""]} + assert handler.descriptions == ["retry test", "retry test (Attempt 1)"] + + +async def test_fresh_only_policy_rides_the_ambient_floor(): + """The decoupling's point: a fresh-only policy gets transient failures + handled by the ambient floor AND wedged failures escalated — without + restating the floor.""" + node_runs: list[str] = [] + llm = FlakyLLM([ + "planned v1", + FakeOverloadedError("transient blip"), + FakeCorruptedError("wedged"), + "planned v2", + "authored v2", + ]) + backoffs: list[int] = [] + handler = _RecordingIOHandler() + policy = _RecordingFreshPolicy(next_thread_id="fresh-floor-2") + + with _ambient_policy(DefaultRetryPolicy( + _retry_on(FakeOverloadedError), backoff=_recording_backoff(backoffs), max_retries=3, + )): + result = await _run( + _build_graph(llm, node_runs), handler, thread_id="fresh-floor-1", retry=policy, + ) + + assert result["log"] == ["", "plan: planned v2", "author: authored v2"] + # The transient failure was floor-retried in place (author re-ran); the + # wedged failure escalated to a fresh start (whole graph re-ran). + assert node_runs == ["plan", "author", "author", "plan", "author"] + assert backoffs == [0] + assert len(policy.rebuilds) == 1 + + +async def test_fresh_start_exhaustion_raises_without_final_rebuild(): + node_runs: list[str] = [] + llm = FlakyLLM([ + "planned v1", + FakeCorruptedError("wedged, first time"), + "planned v2", + FakeCorruptedError("wedged, second time"), + ]) + handler = _RecordingIOHandler() + policy = _RecordingFreshPolicy(next_thread_id="fresh-exhausted-2") + assert policy.max_fresh_retries == 2 + + with pytest.raises(FakeCorruptedError, match="second time"): + await _run(_build_graph(llm, node_runs), handler, thread_id="fresh-exhausted-1", retry=policy) + + assert node_runs == ["plan", "author", "plan", "author"] + # Rebuilt once, between the attempts — the final failure propagates + # without a pointless rebuild of an input that would never run. + assert len(policy.rebuilds) == 1 diff --git a/tests/test_harness_compile_check.py b/tests/test_harness_compile_check.py new file mode 100644 index 00000000..15cd307f --- /dev/null +++ b/tests/test_harness_compile_check.py @@ -0,0 +1,184 @@ +"""Unit tests for the harness compile gate (``composer.spec.source.harness``).""" + +import asyncio +import json + +import pytest +from pydantic import ValidationError + +from composer.spec.source.harness import ForgeReport, _compile_check + +ABSTRACT_ERROR = { + "severity": "error", + "errorCode": "3656", + "message": 'Contract "TokenInstance1" should be marked as abstract.', + "formattedMessage": ( + 'TypeError: Contract "TokenInstance1" should be marked as abstract.\n' + " --> certora/harnesses/TokenInstance1.sol:7:1:\n" + "Note: Missing implementation: \n" + " --> src/interfaces/IToken.sol:543:5:\n" + ), +} + +TRANSIENT_WARNING = { + "severity": "warning", + "errorCode": "2394", + "message": "Transient storage can break composability", + "formattedMessage": "Warning: Transient storage can break composability", +} + + +def _report(*diagnostics: dict) -> str: + return json.dumps({ + "errors": list(diagnostics), + "sources": {}, + "contracts": {}, + "build_infos": [], + }) + + +def test_report_keeps_errors_and_drops_warnings(): + report = ForgeReport.model_validate_json(_report(TRANSIENT_WARNING, ABSTRACT_ERROR)) + assert report.compile_errors == [ABSTRACT_ERROR["formattedMessage"]] + + +def test_report_falls_back_to_the_bare_message(): + report = ForgeReport.model_validate_json( + _report({"severity": "error", "message": "Source not found"}) + ) + assert report.compile_errors == ["Source not found"] + + +def test_report_empty_on_clean_build(): + assert ForgeReport.model_validate_json(_report(TRANSIENT_WARNING)).compile_errors == [] + assert ForgeReport.model_validate_json('{"sources": {}}').compile_errors == [] + + +def test_report_rejects_output_that_is_not_a_report(): + # forge failed before compiling — nothing to hold the harnesses against. + for output in ("Error: failed to resolve remappings", "", "[1, 2]"): + with pytest.raises(ValidationError): + ForgeReport.model_validate_json(output) + + +@pytest.mark.asyncio +async def test_compile_check_skipped_without_a_foundry_project(tmp_path): + assert await _compile_check(str(tmp_path), ["certora/harnesses/TokenInstance1.sol"]) is None + + +@pytest.mark.asyncio +async def test_compile_check_builds_the_delivered_harnesses(tmp_path, monkeypatch): + """forge is invoked on the delivered paths, inside the directory it was + handed — the materialized project, whose layout the paths already match.""" + import composer.spec.source.harness as harness_mod + + (tmp_path / "foundry.toml").write_text("[profile.default]\n") + monkeypatch.setattr(harness_mod.shutil, "which", lambda _: "/usr/local/bin/forge") + invocations = [] + + class _Proc: + async def communicate(self): + return _report(ABSTRACT_ERROR).encode(), b"" + + async def _fake_exec(*cmd, **kwargs): + invocations.append((cmd, kwargs["cwd"])) + return _Proc() + + monkeypatch.setattr(asyncio, "create_subprocess_exec", _fake_exec) + + result = await _compile_check( + str(tmp_path), + ["certora/harnesses/TokenInstance2.sol", "certora/harnesses/TokenInstance1.sol"], + ) + assert result is not None + assert "certora/harnesses/TokenInstance1.sol" in result + (cmd, cwd) = invocations[0] + assert cmd[1:] == ( + "build", + "--json", + "certora/harnesses/TokenInstance1.sol", + "certora/harnesses/TokenInstance2.sol", + ) + assert str(cwd) == str(tmp_path) + + +@pytest.mark.asyncio +async def test_compile_check_accepts_a_clean_build(tmp_path, monkeypatch): + import composer.spec.source.harness as harness_mod + + (tmp_path / "foundry.toml").write_text("[profile.default]\n") + monkeypatch.setattr(harness_mod.shutil, "which", lambda _: "/usr/local/bin/forge") + + class _Proc: + async def communicate(self): + return _report(TRANSIENT_WARNING).encode(), b"" + + async def _fake_exec(*cmd, **kwargs): + return _Proc() + + monkeypatch.setattr(asyncio, "create_subprocess_exec", _fake_exec) + + assert await _compile_check(str(tmp_path), ["certora/harnesses/TokenInstance1.sol"]) is None + + +@pytest.mark.asyncio +async def test_result_tool_sees_the_vfs_without_exposing_it(tmp_path): + """Pins the contract the gate depends on: an ``AsyncResultTool`` that also + carries injected state reads the VFS in its validator, while the model + still sees only ``value``. A rejection is returned to the model and leaves + the result unset; an acceptance stores the validated model. + """ + from typing import NotRequired, override + + from langchain_core.messages import AIMessage + from langgraph.graph import END, START, MessagesState, StateGraph + from langgraph.prebuilt import ToolNode + from pydantic import BaseModel + + from composer.spec.natspec.async_result import AsyncResultTool + from graphcore.tools.schemas import WithInjectedState + from graphcore.tools.vfs import VFSState + + class _Result(BaseModel): + note: str + + class _State(MessagesState, VFSState): + result: NotRequired[_Result] + + vfs_seen = [] + + class _ResultTool(AsyncResultTool[_Result], WithInjectedState[_State]): + """Signal the completion of your workflow.""" + + @override + async def validate_result(self, res: _Result) -> str | None: + vfs_seen.append(sorted(self.state["vfs"])) + return None if res.note == "ok" else f"rejected: {res.note}" + + tool = _ResultTool.as_tool("result") + assert list(tool.args) == ["value"] + + graph = StateGraph(_State) + graph.add_node("tools", ToolNode([tool])) + graph.add_edge(START, "tools") + graph.add_edge("tools", END) + app = graph.compile() + + def _call(note: str, call_id: str) -> dict: + return { + "messages": [AIMessage(content="", tool_calls=[{ + "name": "result", + "args": {"value": {"note": note}}, + "id": call_id, + "type": "tool_call", + }])], + "vfs": {"certora/harnesses/TokenInstance1.sol": "contract TokenInstance1 is Token { }"}, + } + + rejected = await app.ainvoke(_call("bad", "c1")) + assert rejected["messages"][-1].content == "rejected: bad" + assert "result" not in rejected + assert vfs_seen == [["certora/harnesses/TokenInstance1.sol"]] + + accepted = await app.ainvoke(_call("ok", "c2")) + assert accepted["result"] == _Result(note="ok") diff --git a/tests/test_import_diagnostics.py b/tests/test_import_diagnostics.py new file mode 100644 index 00000000..f32ecb1e --- /dev/null +++ b/tests/test_import_diagnostics.py @@ -0,0 +1,205 @@ +"""Tests for classifying `ParserError: Source "…" not found` against the conf's packages. + +Two independent halves: parsing the source unit names (and importers) out of wrap-prone solc +output, and deciding — from the packages list plus the filesystem — which cause each one is. +""" + +from pathlib import Path + +from certora_autosetup.utils.import_diagnostics import ( + UnresolvedImportKind, + classify_unresolved_import, + describe_unresolved_imports, + parse_unresolved_imports, +) +from tests.test_compilation_workarounds import ( + SINGLE_LINE_SOURCE_NOT_FOUND, + UNRELATED_OUTPUT, + WRAPPED_SOURCE_NOT_FOUND_SPLIT_AT_FILE, + WRAPPED_SOURCE_NOT_FOUND_SPLIT_AT_QUOTE, +) + +# solc's own shape: the diagnostic first, the importing file on the following `-->` line. +SOURCE_NOT_FOUND_WITH_IMPORTER = ( + 'ParserError: Source "@vault/core/contracts/IVault.sol" not found: File not found.\n' + " --> smart-contracts/src/Widget.sol:5:1:\n" + " |\n" + '5 | import "@vault/core/contracts/IVault.sol";\n' +) + +TWO_SOURCES_NOT_FOUND = ( + 'ParserError: Source "@vault/core/contracts/IVault.sol" not found: File not found.\n' + " --> src/Widget.sol:5:1:\n" + 'ParserError: Source "solady/utils/FixedPointMathLib.sol" not found: File not found.\n' + " --> src/Vault.sol:7:1:\n" +) + + +def test_parses_wrap_split_at_quote() -> None: + assert parse_unresolved_imports(WRAPPED_SOURCE_NOT_FOUND_SPLIT_AT_QUOTE) == [ + ("@openzeppelin/contracts/token/ERC20/IERC20.sol", None) + ] + + +def test_parses_wrap_split_at_file() -> None: + assert parse_unresolved_imports(WRAPPED_SOURCE_NOT_FOUND_SPLIT_AT_FILE) == [ + ("solady/utils/FixedPointMathLib.sol", None) + ] + + +def test_parses_single_line_form() -> None: + assert parse_unresolved_imports(SINGLE_LINE_SOURCE_NOT_FOUND) == [("src/Foo.sol", None)] + + +def test_parses_nothing_from_unrelated_output() -> None: + assert parse_unresolved_imports(UNRELATED_OUTPUT) == [] + + +def test_associates_the_importer_from_the_source_location_line() -> None: + assert parse_unresolved_imports(SOURCE_NOT_FOUND_WITH_IMPORTER) == [ + ("@vault/core/contracts/IVault.sol", "smart-contracts/src/Widget.sol") + ] + + +def test_each_error_keeps_its_own_importer() -> None: + assert parse_unresolved_imports(TWO_SOURCES_NOT_FOUND) == [ + ("@vault/core/contracts/IVault.sol", "src/Widget.sol"), + ("solady/utils/FixedPointMathLib.sol", "src/Vault.sol"), + ] + + +def test_missing_target_directory_is_a_package_target_miss(tmp_path: Path) -> None: + # The remapping fired (the source unit lives under its target) and the target does not + # exist: the dependency is not installed there. This is the class the ancestor walk fixes. + packages = ["@vault/=smart-contracts/node_modules/@vault/core/"] + + failure = classify_unresolved_import( + "smart-contracts/node_modules/@vault/core/IVault.sol", packages, tmp_path + ) + + assert failure.kind == UnresolvedImportKind.PACKAGE_TARGET_MISSING + assert failure.package_key == "@vault/" + + +def test_installed_package_without_the_remapped_subdirectory_is_its_own_class( + tmp_path: Path, +) -> None: + # `resolve_node_modules_target` already decided this case (kind `subpath_missing`) by finding + # the package directory, so classifying it as "not installed anywhere up to the run root" + # would contradict the resolver and name a remedy that cannot apply. + (tmp_path / "node_modules" / "@oz" / "contracts").mkdir(parents=True) + packages = ["@oz/=node_modules/@oz/contracts/token/"] + + failure = classify_unresolved_import( + "node_modules/@oz/contracts/token/ERC20.sol", packages, tmp_path + ) + + assert failure.kind == UnresolvedImportKind.PACKAGE_SUBPATH_MISSING + assert failure.package_root == str(tmp_path / "node_modules/@oz/contracts") + described = describe_unresolved_imports([failure]) + assert "the package is installed at" in described + assert "not found in any ancestor" not in described + + +def test_missing_package_directory_stays_a_package_target_miss(tmp_path: Path) -> None: + # Same target shape, nothing installed: the class the ancestor walk resolves, unchanged. + packages = ["@oz/=node_modules/@oz/contracts/token/"] + + failure = classify_unresolved_import( + "node_modules/@oz/contracts/token/ERC20.sol", packages, tmp_path + ) + + assert failure.kind == UnresolvedImportKind.PACKAGE_TARGET_MISSING + assert failure.package_root is None + + +def test_missing_non_node_modules_target_stays_a_package_target_miss(tmp_path: Path) -> None: + # forge/soldeer targets have no package root to fall back on, so they keep the plain class. + failure = classify_unresolved_import( + "lib/openzeppelin/contracts/token/ERC20.sol", + ["@oz/=lib/openzeppelin/contracts/"], + tmp_path, + ) + + assert failure.kind == UnresolvedImportKind.PACKAGE_TARGET_MISSING + + +def test_existing_target_directory_is_a_file_miss(tmp_path: Path) -> None: + # The package is installed; the file inside it is what is missing, so rebuilding the + # packages list provably cannot help. + (tmp_path / "node_modules" / "@vault" / "core").mkdir(parents=True) + packages = ["@vault/=node_modules/@vault/core/"] + + failure = classify_unresolved_import( + "node_modules/@vault/core/IVault.sol", packages, tmp_path + ) + + assert failure.kind == UnresolvedImportKind.FILE_MISSING_IN_PACKAGE + + +def test_absolute_and_relative_spellings_of_the_same_target_both_match(tmp_path: Path) -> None: + (tmp_path / "node_modules" / "@vault" / "core").mkdir(parents=True) + absolute = [f"@vault/={tmp_path / 'node_modules/@vault/core'}/"] + relative = ["@vault/=node_modules/@vault/core/"] + source_unit = str(tmp_path / "node_modules/@vault/core/IVault.sol") + + from_absolute = classify_unresolved_import(source_unit, absolute, tmp_path) + from_relative = classify_unresolved_import(source_unit, relative, tmp_path) + + assert from_absolute.kind == UnresolvedImportKind.FILE_MISSING_IN_PACKAGE + assert from_relative.kind == UnresolvedImportKind.FILE_MISSING_IN_PACKAGE + + +def test_import_no_entry_covers_is_unmapped(tmp_path: Path) -> None: + failure = classify_unresolved_import( + "@vault/core/IVault.sol", ["@widget/=lib/widget/"], tmp_path + ) + + assert failure.kind == UnresolvedImportKind.UNMAPPED_IMPORT + assert failure.package_key is None + + +def test_project_tree_source_is_a_missing_project_file(tmp_path: Path) -> None: + (tmp_path / "src").mkdir() + + failure = classify_unresolved_import("src/Widget.sol", [], tmp_path) + + assert failure.kind == UnresolvedImportKind.MISSING_PROJECT_FILE + + +def test_context_scoped_key_that_misses_the_importer_carries_a_hint(tmp_path: Path) -> None: + # The key covers the import textually, so the remapping was declared — its context is why + # it never applied. That needs the importer, which solc does not always print, so it is a + # hint on an UNMAPPED_IMPORT rather than a kind of its own. + packages = ["src/Widget/:@vault/=src/Widget/dependencies/vault/"] + + failure = classify_unresolved_import( + "@vault/IVault.sol", packages, tmp_path, importer="smart-contracts/src/Widget/Widget.sol" + ) + + assert failure.kind == UnresolvedImportKind.UNMAPPED_IMPORT + assert failure.hint is not None + assert "src/Widget/" in failure.hint + + +def test_description_names_the_remedy_per_kind(tmp_path: Path) -> None: + (tmp_path / "node_modules" / "@widget" / "lib").mkdir(parents=True) + failures = [ + classify_unresolved_import( + "node_modules/@vault/core/IVault.sol", ["@vault/=node_modules/@vault/core/"], tmp_path + ), + classify_unresolved_import( + "node_modules/@widget/lib/IWidget.sol", ["@widget/=node_modules/@widget/lib/"], tmp_path + ), + ] + + described = describe_unresolved_imports(failures) + + assert "package_target_missing (1):" in described + assert "file_missing_in_package (1):" in described + assert "not found in any ancestor node_modules" in described + assert "rebuilding the packages list cannot help" in described + + +def test_description_of_nothing_is_empty() -> None: + assert describe_unresolved_imports([]) == "" diff --git a/tests/test_migration_oracle.py b/tests/test_migration_oracle.py index 1fd5f296..f78bffb7 100644 --- a/tests/test_migration_oracle.py +++ b/tests/test_migration_oracle.py @@ -29,7 +29,7 @@ from composer.rag.models import DefaultEmbedder from composer.spec.agent_index import AgentIndex, AgentIndexConfig from composer.spec.source.munge.edit_oracle import mk_oracle -from composer.spec.source.munge.edit_store import StoredEdit +from composer.spec.source.munge.edit_store import MungeEditor, StoredEdit from composer.spec.source.versioned_index import ( VersionedAgentIndex, AnswerPortability, Stale, UpToDate, ) @@ -61,7 +61,10 @@ def _sc(project_root) -> Any: def _stored(vfs: dict[str, str]) -> StoredEdit: - return StoredEdit(vfs=vfs, executive_summary="summary", why_sound="sound") + return StoredEdit( + vfs=vfs, executive_summary="summary", why_sound="sound", + attribution=MungeEditor(), + ) @pytest.mark.asyncio diff --git a/tests/test_null_solana_backend.py b/tests/test_null_solana_backend.py index 986bff15..7a5e2854 100644 --- a/tests/test_null_solana_backend.py +++ b/tests/test_null_solana_backend.py @@ -85,12 +85,12 @@ async def test_formalize_echoes_properties_into_result(): props = _props() result = await NullSolanaFormalizer().formalize( - "batch", feat, props, cast(Any, None), cast(Any, None) + "batch", feat, props, cast(Any, None), cast(Any, None), cast(Any, None) ) assert isinstance(result, NullResult) # Every property is echoed back verbatim as its own single-rule mapping. - assert result.property_units() == [ + assert result.property_checks() == [ ("balance_conserved", ["balance_conserved"]), ("only_authority_withdraws", ["only_authority_withdraws"]), ] @@ -109,10 +109,10 @@ async def test_formalize_echoes_properties_into_result(): @pytest.mark.asyncio async def test_formalize_with_no_properties_records_empty(): result = await NullSolanaFormalizer().formalize( - "batch", _unit(), [], cast(Any, None), cast(Any, None) + "batch", _unit(), [], cast(Any, None), cast(Any, None), cast(Any, None) ) assert isinstance(result, NullResult) - assert result.property_units() == [] + assert result.property_checks() == [] assert "0 properties" in result.commentary @@ -128,7 +128,7 @@ async def test_prepare_system_locates_main_and_builds_formalizer(tmp_path): backend = _backend(str(tmp_path)) run = cast(Any, SimpleNamespace(source=SimpleNamespace(contract_name="vault"))) - prepared = await backend.prepare_system(feat.app, run) + prepared = await backend.prepare_system(feat.app, run, await backend.preflight(run)) assert isinstance(prepared, NullSolanaPrepared) # prepare_system routes through SOLANA.locate_main, so main is the matched program. diff --git a/tests/test_pipeline_overlap.py b/tests/test_pipeline_overlap.py new file mode 100644 index 00000000..7463d2be --- /dev/null +++ b/tests/test_pipeline_overlap.py @@ -0,0 +1,280 @@ +"""Tests for the driver's two overlaps — the build-shaped steps run alongside the LLM steps that +don't depend on them. + +``run_pipeline`` overlaps twice (``composer/pipeline/core.py``): + +* ``backend.preflight`` (Crucible's program build + harness-skeleton build) with **system analysis**; +* ``prepared.prepare_formalization`` (the prover's autosetup) with **property extraction**. + +Neither side of either pair needs the other. The preflight is additionally a *gate*: it shares a task +group with the analysis, so a failure on either side cancels the other rather than letting it spend +on a run that can no longer complete. Nothing else is cancelled — the second pair is awaited in turn. + +Stubs throughout — no LLM, no DB, no backend wheel. +""" + +import asyncio + +import pytest + +import composer.pipeline.core as core +from composer.pipeline.core import run_pipeline +from composer.pipeline.ecosystem import EVM + +# The driver needs *an* ecosystem to reach the overlap under test, but never exercises this one: +# the analysis and extraction it feeds are both monkeypatched below. EVM is the convenient real +# value — ``supports_greenfield=True`` clears the driver's greenfield assert without a live +# ``env``, and its ``analysis_extra_input`` reads only the two fields ``_Source`` supplies. +ECOSYSTEM = EVM + +pytestmark = pytest.mark.asyncio + +#: What a successful stub preflight hands forward to ``prepare_system``. +PREFLIGHT_RESULT = "prepared-workspace" + + +class _Store: + def write_properties(self, *_a, **_kw): ... + def write_artifact(self, *_a, **_kw): return "artifact" + def write_report(self, *_a, **_kw): ... + + +class _Step: + """A stub async step that takes ``delay`` seconds and then fails or returns ``value``.""" + + def __init__(self, delay: float, error: Exception | None, value: object = None): + self.delay, self.error, self.value = delay, error, value + self.finished = False + self.cancelled = False + + async def run(self): + try: + await asyncio.sleep(self.delay) + except asyncio.CancelledError: + self.cancelled = True + raise + if self.error is not None: + raise self.error + self.finished = True + return self.value + + +class _Prepared: + """A prepared system whose ``prepare_formalization`` runs one :class:`_Step`.""" + + main = "main-unit" + + def __init__(self, step: _Step): + self.step = step + + async def prepare_formalization(self, _run): + return await self.step.run() + + +class _Backend: + analysis_spec = core.SystemAnalysisSpec("analysis-key", "properties-key") + core_phases = {"analysis": 1, "extraction": 2, "formalization": 3, "report": 4} + backend_guidance = "guidance" + artifact_store = _Store() + + def __init__(self, prepared: _Prepared, preflight: _Step | None = None): + self._prepared = prepared + self.preflight_step = preflight or _Step(0, None, PREFLIGHT_RESULT) + #: What the driver handed to ``prepare_system`` — the preflight's own result. + self.seen_preflight: object = None + + async def preflight(self, _run): + return await self.preflight_step.run() + + async def prepare_system(self, _analyzed, _run, preflight): + self.seen_preflight = preflight + return self._prepared + + def to_artifact_id(self, _c): return "artifact-id" + + +class _Ctx: + """A workflow context that hands back itself for any child scope.""" + + recursion_limit = 10 + + def child(self, *_a, **_kw): + return self + + +class _Source: + """The two source fields the shared front half puts in the analysis prompt.""" + + contract_name = "Counter" + relative_path = "src/Counter.sol" + + +class _Run: + """Just enough ``PipelineRun`` for the driver: runners that await the job inline.""" + + source = _Source() + env = None + ctx = _Ctx() + + async def runner(self, _task_info, job): + return await job() + + async def cpu_runner(self, _task_info, job): + return await job() + + +async def _drive( + monkeypatch, + *, + prep: _Step, + extract: _Step, + analysis: _Step | None = None, + preflight: _Step | None = None, +) -> dict: + """Run the driver with stubbed analysis + extraction; report the outcome and the backend.""" + analysis = analysis or _Step(0, None, "analyzed") + + async def fake_analysis(*_a, **_kw): + return await analysis.run() + + async def fake_extract_all(*_a, **_kw): + await extract.run() + return [] # no batches — the driver's own "nothing extracted" error, if it gets that far + + monkeypatch.setattr(core, "run_component_analysis", fake_analysis) + monkeypatch.setattr(core, "_extract_all", fake_extract_all) + backend = _Backend(_Prepared(prep), preflight) + seen: dict = {"backend": backend} + try: + await run_pipeline(backend, _Run(), max_bug_rounds=1, ecosystem=ECOSYSTEM) # type: ignore[arg-type] + except BaseException as exc: # noqa: BLE001 — the outcome under test + seen["raised"] = exc + return seen + + +# --------------------------------------------------------------------------- +# preflight ∥ system analysis +# --------------------------------------------------------------------------- + + +async def test_a_preflight_failure_stops_system_analysis(monkeypatch): + # The whole point of gating the workspace this early: a toolchain failure must not wait out the + # analysis agent already running beside it (and must never reach extraction, which is where the + # real spend is). + boom = RuntimeError("the harness workspace does not build") + analysis = _Step(30, None, "analyzed") + extract = _Step(30, None) + seen = await _drive( + monkeypatch, prep=_Step(0, None), extract=extract, + analysis=analysis, preflight=_Step(0.01, boom), + ) + + assert seen["raised"] is boom + assert analysis.cancelled and not analysis.finished + # Extraction never even started: it is created only after prepare_system. + assert not extract.finished and not extract.cancelled + + +async def test_an_analysis_failure_stops_the_preflight(monkeypatch): + # Symmetric: the gate spends no money, but it is the run's slowest non-LLM step, so an analysis + # that has already failed must not wait out a workspace build whose result nothing will read. + boom = RuntimeError("system analysis blew up") + preflight = _Step(30, None, PREFLIGHT_RESULT) + seen = await _drive( + monkeypatch, prep=_Step(0, None), extract=_Step(0, None), + analysis=_Step(0.01, boom), preflight=preflight, + ) + + assert seen["raised"] is boom + assert preflight.cancelled and not preflight.finished + + +async def test_a_double_failure_reports_both(monkeypatch): + # The one case the driver cannot answer with a single error: both sides raise before either + # cancellation lands, so neither is "the" cause and the group carries them both. + analysis_boom = RuntimeError("system analysis blew up") + preflight_boom = RuntimeError("the harness workspace does not build") + seen = await _drive( + monkeypatch, prep=_Step(0, None), extract=_Step(0, None), + analysis=_Step(0, analysis_boom), preflight=_Step(0, preflight_boom), + ) + + raised = seen["raised"] + assert isinstance(raised, BaseExceptionGroup) + assert set(raised.exceptions) == {analysis_boom, preflight_boom} + + +async def test_cancelling_the_run_cancels_the_steps_it_was_waiting_on(monkeypatch): + # When the *caller* goes away (Ctrl-C, an enclosing timeout), both overlapped steps must go with + # it: the awaited one comes along for free, but the analysis the driver is not sitting on would + # keep running detached — a multi-minute agent outliving the run that started it. + analysis = _Step(30, None, "analyzed") + preflight = _Step(30, None, PREFLIGHT_RESULT) + + async def fake_analysis(*_a, **_kw): + return await analysis.run() + + async def fake_extract_all(*_a, **_kw): + return [] + + monkeypatch.setattr(core, "run_component_analysis", fake_analysis) + monkeypatch.setattr(core, "_extract_all", fake_extract_all) + driver = asyncio.create_task( + run_pipeline( # type: ignore[arg-type] + _Backend(_Prepared(_Step(0, None)), preflight), _Run(), + max_bug_rounds=1, ecosystem=ECOSYSTEM, + ) + ) + await asyncio.sleep(0.05) # both overlapped steps are now in flight + driver.cancel() + with pytest.raises(asyncio.CancelledError): + await driver + + assert analysis.cancelled and not analysis.finished + assert preflight.cancelled and not preflight.finished + + +async def test_the_preflight_result_is_handed_to_prepare_system(monkeypatch): + # The backend's prep travels forward as a value, so a backend can build on it as immutable state + # instead of stashing it on itself between the two calls. + seen = await _drive(monkeypatch, prep=_Step(0, None), extract=_Step(0, None)) + assert seen["backend"].seen_preflight == PREFLIGHT_RESULT + + +# --------------------------------------------------------------------------- +# prepare_formalization ∥ property extraction +# --------------------------------------------------------------------------- + + +async def test_a_setup_failure_ends_the_run_once_extraction_is_done(monkeypatch): + # This pair is only overlapped, not gated: extraction runs to completion, and setup's failure is + # then reported as itself — not as a downstream "no properties extracted", which is what an + # entirely *unobserved* setup failure would look like. + boom = RuntimeError("cargo-build-sbf failed (exit 101)") + extract = _Step(0.03, None) + seen = await _drive(monkeypatch, prep=_Step(0.01, boom), extract=extract) + + assert seen["raised"] is boom + assert extract.finished and not extract.cancelled + + +async def test_an_extraction_failure_ends_the_run_with_its_own_error(monkeypatch): + # The other direction: extraction is awaited first, so its failure is what surfaces. + boom = RuntimeError("extraction blew up") + prep = _Step(0, None) + seen = await _drive(monkeypatch, prep=prep, extract=_Step(0.01, boom)) + + assert seen["raised"] is boom + assert prep.finished + + +async def test_both_succeeding_still_reaches_the_drivers_own_checks(monkeypatch): + # The overlap is preserved: nothing is cancelled when both sides are fine, and the driver's + # "nothing extracted" guard is what speaks for an empty extraction. + prep, extract = _Step(0.01, None), _Step(0.01, None) + seen = await _drive(monkeypatch, prep=prep, extract=extract) + + assert extract.finished and not extract.cancelled + assert prep.finished + assert isinstance(seen["raised"], ValueError) + assert "No properties extracted" in str(seen["raised"]) diff --git a/tests/test_pipeline_result.py b/tests/test_pipeline_result.py index ee8f0a86..a241d50b 100644 --- a/tests/test_pipeline_result.py +++ b/tests/test_pipeline_result.py @@ -2,15 +2,16 @@ ``CorePipelineResult.all_failed`` is the signal the autoprove/foundry entry points translate into a non-zero exit code: a run in which *every* attempted component failed -to generate a deliverable — either it gave up (``GaveUp``) or it crashed -(``BaseException``) — is a total failure. As long as one component delivered, the run -succeeds regardless of how many others gave up. +to generate a deliverable — it gave up (``GaveUp``), crashed (``BaseException``), or was +cut short by the budget (``Curtailed``, whose partial is not a reliable deliverable) — +is a total failure. As long as one component delivered, the run succeeds regardless of +how many others gave up. """ from pathlib import Path from typing import Any, cast from composer.pipeline.ptypes import ( - ComponentOutcome, CorePipelineResult, Delivered, GaveUp, + ComponentOutcome, CorePipelineResult, Curtailed, Delivered, GaveUp, ) from composer.spec.system_model import ContractComponentInstance @@ -21,11 +22,19 @@ def _delivered() -> Delivered: return Delivered(result=cast(Any, None), deliverable=Path("composer_c.spec")) -def _outcome(result: Delivered | GaveUp | BaseException) -> ComponentOutcome: +def _curtailed_with_partial() -> Curtailed[Delivered]: + return Curtailed(Delivered(result=cast(Any, None), + deliverable=Path("composer_c.spec.unverified"))) + + +type _Outcome = Delivered | Curtailed[Delivered] | GaveUp | BaseException + + +def _outcome(result: _Outcome) -> ComponentOutcome: return ComponentOutcome(feat=cast(ContractComponentInstance, None), props=[], result=result) -def _result(*results: Delivered | GaveUp | BaseException) -> CorePipelineResult: +def _result(*results: _Outcome) -> CorePipelineResult: outcomes = [_outcome(r) for r in results] return CorePipelineResult( n_components=len(outcomes), n_properties=0, outcomes=outcomes, failures=[], @@ -61,6 +70,20 @@ def test_giveup_and_crash_mix_with_no_delivery_is_a_failure(): assert r.all_failed is True +def test_curtailed_partial_is_not_a_delivery(): + # A budget-curtailed component published something, but under lifted gates — + # it must not count as delivered, so an all-curtailed run is a total failure. + r = _result(_curtailed_with_partial(), Curtailed(None)) + assert r.n_delivered == 0 + assert r.all_failed is True + + +def test_one_delivered_among_curtailed_is_not_a_failure(): + r = _result(_delivered(), _curtailed_with_partial(), Curtailed(None, detail="stopped")) + assert r.n_delivered == 1 + assert r.all_failed is False + + def test_empty_outcomes_is_not_reported_as_failure(): # The driver raises before returning an empty outcome set; the guard keeps # "all of nothing" from being reported as a total failure regardless. diff --git a/tests/test_pipeline_staged_formalizer.py b/tests/test_pipeline_staged_formalizer.py index afdfc878..c5fe39a3 100644 --- a/tests/test_pipeline_staged_formalizer.py +++ b/tests/test_pipeline_staged_formalizer.py @@ -46,7 +46,7 @@ class _Result: artifact_text = "" unit_file = None run_link = None - def property_units(self): return [] + def property_checks(self): return [] class _Formalizer: @@ -59,7 +59,7 @@ class _Formalizer: def __init__(self, calls: list[tuple[str, list[str]]] | None = None): self.calls = [] if calls is None else calls - async def formalize(self, _label, feat, props, _ctx, _run): + async def formalize(self, _label, feat, props, _ctx, _run, _extra_tools): # A yield point, so a driver that started the fan-out before `begin` finished would # interleave here and be caught by the ordering assertion. await asyncio.sleep(0) @@ -100,7 +100,9 @@ class _Backend: def __init__(self, prepared): self._prepared = prepared - async def prepare_system(self, _analyzed, _run): return self._prepared + async def preflight(self, _run): return None + + async def prepare_system(self, _analyzed, _run, _preflight): return self._prepared def to_artifact_id(self, _c): return "artifact-id" @@ -109,17 +111,28 @@ class _Cache: async def cache_get(self, _ty): return None async def cache_put(self, _v): return None + def child(self, key, tags = None): + if tags is None: + return _Cache() + async def thunk(): + return _Cache() + return thunk() + + class _Ctx: recursion_limit = 10 def child(self, *_a, **_kw): return self - async def achild(self, *_a, **_kw): return _Cache() - class _FeatCtx: - async def child(self, *_a, **_kw): return _Cache() + def child(self, key, tags = None): + if tags is None: + return _Cache() + async def thunk(): + return _Cache() + return thunk() class _Source: @@ -132,7 +145,7 @@ class _Run: env = None ctx = _Ctx() - async def runner(self, _task_info, job): + async def runner(self, task_info, job): return await job() diff --git a/tests/test_prover_nag_integration.py b/tests/test_prover_nag_integration.py new file mode 100644 index 00000000..f9ad68d0 --- /dev/null +++ b/tests/test_prover_nag_integration.py @@ -0,0 +1,158 @@ +"""End-to-end integration test for the stuck-rule nag ("prover nagging") +behavior of ``verify_spec``. + +Runs the full autoprove pipeline on the Counter scenario with the nag-variant +tape (``install_nag_tape``). The prover core itself is mocked — the nag +machinery under test lives entirely in ``verify_spec``'s post-processing, +above the ``run_prover`` seam — so no cloud (or local) prover jobs run: +``_fake_run_prover`` reads the spec each call targets and reports every +declared rule VERIFIED except the permanently-stuck one, which reports +SANITY_FAILED (the status the real ``rule_sanity`` check would produce for +its vacuous body). + +The taped author runs the spec ``STUCK_RULE_NAG_THRESHOLD`` times, nudging it +with a trailing comment between runs (the streak keys on (rule, status), not +spec digest — and the distinct digests stay clear of verify_spec's +identical-spec re-run gate); the last run must append a ``NagMarker`` to +``prover_history`` and queue the stuck-rule reminder, which the author monitor +injects into the conversation as a ```` HumanMessage. The +tape then reacts as the reminder suggests (marks the rule expected-to-fail), +re-verifies, and publishes. + +Pass/fail: the pipeline completes without raising, AND the reminder actually +reached the author's prompt. The fake LLM is the only observer of the real +conversation, so delivery is asserted by a ``HarnessFakeLLM`` subclass that +sniffs every prompt it is asked to answer for ```` blocks — +exactly once per prompt (a re-fired nag or a non-draining reminders channel +would stack duplicates). + +Still marked ``expensive``: no prover money is spent, but the run needs the +testcontainer Postgres and the real local CVL toolchain (``put_cvl_raw``'s +Typechecker gate), which puts it well outside the routine fast pass. Run with +``-m expensive``. +""" +from pathlib import Path +from typing import Any, override + +import pytest +from pydantic import Field + +from langchain_core.messages import AIMessage, HumanMessage +from langchain_core.prompt_values import PromptValue + +from composer.diagnostics.timing import RunSummary, get_current_task_id +from composer.pipeline.core import formalize_task_id +from composer.prover.core import ProverReport +from composer.prover.ptypes import RulePath, StatusCodes +from composer.spec.source.autoprove_common import autoprove_executor +from composer.testing.harness_tape import HarnessFakeLLM +from composer.testing.ui_harness_autoprove_Counter import ( + NAG_STUCK_RULE, + autoprove_nag_lanes, + install_nag_tape, +) +from composer.ui.autoprove_console import AutoProveConsoleHandler + +from tests.conftest import ( + SPEC_DECL_RE, conf_of_prover_call, needs_postgres, spec_of_prover_conf, +) +from tests.test_autoprove_integration import _install_mocks, _make_args + +pytestmark = [pytest.mark.expensive, needs_postgres, pytest.mark.asyncio] + +_SCENARIO_NAME = "autoprove_counter" + +# Stable prefix of the reminder verify_spec queues when the stuck-rule +# detector fires (the "3" is spelled by the message itself, so match short of it). +_NAG_SNIPPET = "identical failures on the last" + + +async def _fake_run_prover( + folder: Path, args: list[str], tool_call_id: str, + prover_opts: Any, callbacks: Any, cex: Any, +) -> ProverReport: + """Stand-in for ``run_prover``, patched over the binding ``verify_spec`` + calls through (the same seam the ``certora_prover`` conftest fixture + patches). Parses the conf the tool just wrote, reads the spec it verifies, + and reports every declared rule/invariant VERIFIED except the permanently + stuck ``NAG_STUCK_RULE`` → SANITY_FAILED. Content-derived, so both + authoring lanes are served without ordering assumptions.""" + conf = conf_of_prover_call(folder, args) + spec_text = spec_of_prover_conf(folder, conf) + statuses: dict[RulePath, StatusCodes] = { + RulePath(rule=name): ("SANITY_FAILED" if name == NAG_STUCK_RULE else "VERIFIED") + for name in SPEC_DECL_RE.findall(spec_text) + } + assert statuses, f"fake prover: no rule/invariant declarations in {conf['verify']}" + return ProverReport( + raw_rule_status=statuses, + result_str="\n".join(f"{p.rule}: {s}" for p, s in statuses.items()), + link="https://prover.example/fake-run", + certora_run_stdout="" + ) + + +async def _fake_declared_rules(folder: Path, args: list[str]) -> list[str]: + """Stand-in for ``declared_rules_list`` — the same content-derived ground + truth, without the certoraRun build + typechecker ``-listRules`` subprocesses.""" + return SPEC_DECL_RE.findall(spec_of_prover_conf(folder, conf_of_prover_call(folder, args))) + + +class _ReminderSniffingLLM(HarnessFakeLLM): + """Records, per LLM call, the ```` human messages present + in the prompt (keyed by tape lane). The monitor-injected nag reminder lives + in the author's message history, and the fake LLM is the only place the + real conversation can be observed.""" + + sniffed: list[tuple[str, list[str]]] = Field(default_factory=list, exclude=True) + + @override + async def ainvoke(self, input: Any, config: Any = None, *, stop: Any = None, **kwargs: Any) -> AIMessage: + msgs = list(input.to_messages()) if isinstance(input, PromptValue) else list(input) + reminders : list[str] = [ + m.text for m in msgs + if isinstance(m, HumanMessage) and "" in m.text + ] + if reminders: + self.sniffed.append((get_current_task_id() or "", reminders)) + return await super().ainvoke(input, config, stop=stop, **kwargs) + + +async def test_prover_nag_fires_and_run_survives(scenario_provider, langgraph_db, monkeypatch): + scenario_dir = scenario_provider.by_name(_SCENARIO_NAME) + fake = _ReminderSniffingLLM(lanes=autoprove_nag_lanes(), with_human_delay=False) + _install_mocks( + monkeypatch, scenario_dir, tape_installer=lambda: install_nag_tape(fake=fake) + ) + # This test is about verify_spec's monitoring behavior, not proving — swap + # the prover core for the canned status reports, and the rule-listing + # pre-pass for the same spec-derived ground truth. + monkeypatch.setattr("composer.spec.source.prover.run_prover", _fake_run_prover) + monkeypatch.setattr("composer.spec.source.prover.declared_rules_list", _fake_declared_rules) + + # Run the whole pipeline. The first requirement is simply that the nag + # path doesn't kill (or corrupt) the run: this raises if any phase dies. + summary = RunSummary() + async with autoprove_executor( + _make_args(langgraph_db.rag_db, scenario_dir, str(scenario_dir / "system.md")), + summary, + ) as run: + await run(AutoProveConsoleHandler().make_handler) + + # The nag reminder reached the author's conversation... + nag_prompts = [ + (lane, [t for t in texts if _NAG_SNIPPET in t]) + for lane, texts in fake.sniffed + ] + nag_prompts = [(lane, ts) for lane, ts in nag_prompts if ts] + assert nag_prompts, ( + "the stuck-rule nag never reached any prompt; system-reminders seen: " + f"{fake.sniffed}" + ) + # ...in the component-formalization lane, naming the stuck rule... + assert all(lane == formalize_task_id(0) for lane, _ in nag_prompts) + assert all(NAG_STUCK_RULE in t for _, ts in nag_prompts for t in ts) + # ...and exactly once per prompt: the post-skip verify run must not re-nag, + # and the reminders channel must drain on injection — either failure would + # stack a second copy of the reminder into later prompts. + assert all(len(ts) == 1 for _, ts in nag_prompts) diff --git a/tests/test_rag_env.py b/tests/test_rag_env.py new file mode 100644 index 00000000..0d171814 --- /dev/null +++ b/tests/test_rag_env.py @@ -0,0 +1,84 @@ +"""The descriptor-driven RAG corpus registry (``composer.tools.rag_env``). + +A wheel names a corpus by tag (``rag_db_default``); a tag is usable only when *both* halves are +registered — its connection in ``composer.rag.db.KNOWLEDGE_BASES`` and its search-tool factory in +``rag_env._FACTORIES``. The two failure modes are deliberately opposite, and these pin that: + +* an **unregistered tag** is a repo/wheel bug — it raises, at descriptor load, before the run + spends anything; +* an **unavailable corpus** (DB down, embedding model missing) is an environment condition — the + run continues with no RAG surface, because a search aid must never fail a run. + +No corpus is registered on this branch, so the tests that need one register a stub. That stub is +also the executable spec for adding a real one: two entries, in two maps. +""" + +import pytest + +from composer.rag import db as rag_db +from composer.tools import rag_env + + +def _register(monkeypatch: pytest.MonkeyPatch, tag: str, factory) -> None: + """Register both halves of a corpus for one test — the module-level tables are the registry, + so a test corpus goes in the same way a real one would.""" + monkeypatch.setitem(rag_db.KNOWLEDGE_BASES, tag, "postgresql://stub/rag_db") + monkeypatch.setitem(rag_env._FACTORIES, tag, factory) + + +def test_a_wheel_that_declares_no_corpus_is_fine(): + assert rag_env.validate_rag_db(None) is None + + +def test_an_unregistered_tag_raises_and_says_where_to_register_it(): + with pytest.raises(ValueError, match="not a registered RAG corpus") as e: + rag_env.validate_rag_db("no_such_kb") + msg = str(e.value) + assert "KNOWLEDGE_BASES" in msg and "rag_env" in msg + + +def test_the_message_says_so_when_nothing_is_registered_at_all(): + # An empty registry is the intended resting state, and "known: []" would read as a lookup + # failure against a populated one. + assert "none is registered yet" in str( + pytest.raises(ValueError, rag_env.validate_rag_db, "no_such_kb").value + ) + + +def test_half_a_registration_is_not_a_corpus(monkeypatch: pytest.MonkeyPatch): + # Tools but no connection: nothing to search. A half-registration must fail validation, not + # validate and then silently produce no tools. + monkeypatch.setitem(rag_env._FACTORIES, "tools_only", lambda _db: ()) + with pytest.raises(ValueError, match="not a registered RAG corpus"): + rag_env.validate_rag_db("tools_only") + + # …and a connection with no tools factory is just as unusable. + monkeypatch.setitem(rag_db.KNOWLEDGE_BASES, "conn_only", "postgresql://stub/rag_db") + with pytest.raises(ValueError, match="not a registered RAG corpus"): + rag_env.validate_rag_db("conn_only") + + +def test_a_registered_tag_validates(monkeypatch: pytest.MonkeyPatch): + _register(monkeypatch, "stub_kb", lambda _db: ()) + assert rag_env.validate_rag_db("stub_kb") is None + + +def test_building_tools_for_an_unregistered_tag_raises_rather_than_degrading(): + # Not caught by the degrade path: nothing about the environment would make that corpus appear. + with pytest.raises(ValueError, match="not a registered RAG corpus"): + rag_env.build_rag_tools("no_such_kb") + + +def test_an_unavailable_corpus_degrades_to_no_rag(monkeypatch: pytest.MonkeyPatch, caplog): + def explodes(_db): + raise RuntimeError("connection refused") + + _register(monkeypatch, "stub_kb", explodes) + # Stub the embedder: loading a real sentence-transformers model costs seconds, and this test is + # about what happens *after* the corpus is opened. Without this the factory below is never + # reached on a machine with no model installed, and the test would pass for the wrong reason. + monkeypatch.setattr("composer.rag.models.get_model", lambda: None) + with caplog.at_level("WARNING"): + assert rag_env.build_rag_tools("stub_kb") == () + assert "unavailable" in caplog.text + assert "connection refused" in caplog.text # the factory's failure, not the embedder's diff --git a/tests/test_rag_import.py b/tests/test_rag_import.py new file mode 100644 index 00000000..6c560b8d --- /dev/null +++ b/tests/test_rag_import.py @@ -0,0 +1,215 @@ +"""The generic manifest importer (``composer.scripts.rag_import``). + +The importer owns everything a producer deliberately doesn't (see ``docs/rag-import-format.md``): +each of the manifest's two products feeds exactly its own index, ```` tags are +assigned, embedded blocks are cut as their kind dictates, and ``part`` is numbered per header +path — across sections *and* across manifests that resolve to the same DB, because +``manual_sections`` is unique on ``(h1..h6, part)``. These pin that contract without a DB. + +Skips where the ``ragbuild`` group isn't installed: the importer pulls in spaCy transitively +(``text_processors``), which the routine test env doesn't otherwise need. +""" + +import asyncio +import json +import pathlib + +import pytest + +spacy = pytest.importorskip("spacy") + +from composer.rag.import_format import ( # noqa: E402 + EmbeddedBlock, + EmbeddedGroup, + ManualBlock, + ManualSection, + RagManifest, +) +from composer.scripts import rag_import # noqa: E402 + + +def _config(max_length: int = 2000) -> "rag_import.BuilderConfig": + """A real ``BuilderConfig`` without a downloaded model — a blank pipeline plus the rule-based + sentencizer is all ``BlockBuilder`` asks of ``nlp`` (it splits on ``.sents``).""" + nlp = spacy.blank("en") + nlp.add_pipe("sentencizer") + return rag_import.BuilderConfig(nlp=nlp, max_length=max_length) + + +class _RecordingDB: + """Records the two ingestion paths instead of writing them.""" + + def __init__(self) -> None: + self.embedded: list[rag_import.BlockChunk] = [] + self.manual: list[rag_import.BlockChunk] = [] + + async def add_chunks_batch(self, chunks: list[rag_import.BlockChunk]) -> None: + self.embedded.extend(chunks) + + async def add_manual_section(self, chunk: rag_import.BlockChunk) -> None: + self.manual.append(chunk) + + +_HEADERS = ["Guide", "Topic"] + + +def _section(*blocks: ManualBlock, headers: list[str] | None = None) -> ManualSection: + return ManualSection(headers=headers if headers is not None else list(_HEADERS), blocks=list(blocks)) + + +def _group(*blocks: EmbeddedBlock, headers: list[str] | None = None) -> EmbeddedGroup: + return EmbeddedGroup(headers=headers if headers is not None else list(_HEADERS), blocks=list(blocks)) + + +def _manifest( + *, + manual: list[ManualSection] | None = None, + embedded: list[EmbeddedGroup] | None = None, + kb: str = "stub_kb", +) -> RagManifest: + return RagManifest(knowledge_base=kb, manual_sections=manual or [], embedded_groups=embedded or []) + + +def _ingest(*manifests: RagManifest, max_length: int = 2000) -> _RecordingDB: + """Drive ``_ingest`` over manifests sharing one DB, as the CLI does per resolved target.""" + db = _RecordingDB() + seen: dict[tuple[str, ...], int] = {} + config = _config(max_length) + + async def run() -> None: + for m in manifests: + await rag_import._ingest(db, m, config, seen) + + asyncio.run(run()) + return db + + +def test_each_product_feeds_exactly_its_own_index(): + db = _ingest( + _manifest( + manual=[_section(ManualBlock(kind="text", body="Seeds are encoded first."))], + embedded=[_group(EmbeddedBlock(kind="paragraph", body="Seeds are encoded first."))], + ) + ) + assert len(db.manual) == 1 + assert len(db.embedded) == 1 + + manual_only = _ingest(_manifest(manual=[_section(ManualBlock(kind="text", body="x"))])) + assert manual_only.manual and not manual_only.embedded + + embedded_only = _ingest(_manifest(embedded=[_group(EmbeddedBlock(kind="paragraph", body="x"))])) + assert embedded_only.embedded and not embedded_only.manual + + +def test_the_manual_chunk_holds_the_whole_section_with_code_as_refs(): + db = _ingest( + _manifest( + manual=[ + _section( + ManualBlock(kind="text", body="Derive the address."), + ManualBlock(kind="code", body="let (pda, bump) = find_program_address(...);"), + ManualBlock(kind="text", body="Then check the bump."), + ) + ] + ) + ) + (manual,) = db.manual + assert manual.chunk == ( + "Derive the address.\n\n\n\nThen check the bump." + ) + assert manual.code_refs == ["let (pda, bump) = find_program_address(...);"] + + +def test_code_refs_are_numbered_per_section_not_per_manifest(): + db = _ingest( + _manifest( + manual=[ + _section(ManualBlock(kind="code", body="first")), + _section(ManualBlock(kind="code", body="second"), headers=["Guide", "Other"]), + ] + ) + ) + assert [m.chunk for m in db.manual] == ["", ""] + assert [m.code_refs for m in db.manual] == [["first"], ["second"]] + + +def test_a_repeated_header_path_bumps_part(): + db = _ingest( + _manifest( + manual=[ + _section(ManualBlock(kind="text", body="one")), + _section(ManualBlock(kind="text", body="two")), + _section(ManualBlock(kind="text", body="three"), headers=["Guide", "Elsewhere"]), + ] + ) + ) + assert [(tuple(m.headers), m.part) for m in db.manual] == [ + (("Guide", "Topic"), 0), + (("Guide", "Topic"), 1), + (("Guide", "Elsewhere"), 0), + ] + + +def test_part_numbering_continues_across_manifests_sharing_a_db(): + # Two manifests, one target: the (headers, part) unique key spans both, so the counter must too. + db = _ingest( + _manifest(manual=[_section(ManualBlock(kind="text", body="one"))]), + _manifest(manual=[_section(ManualBlock(kind="text", body="two"))]), + ) + assert [m.part for m in db.manual] == [0, 1] + + +def test_an_overlong_paragraph_is_split_at_sentence_boundaries(): + body = " ".join(f"Sentence number {i} says something." for i in range(40)) + db = _ingest( + _manifest(embedded=[_group(EmbeddedBlock(kind="paragraph", body=body))]), max_length=120 + ) + assert len(db.embedded) > 1 + assert all(len(c.chunk) < 240 for c in db.embedded) + + +def test_an_overlong_atomic_block_stays_whole(): + body = "\n".join(f"| row {i} | value {i} |" for i in range(40)) + db = _ingest( + _manifest(embedded=[_group(EmbeddedBlock(kind="atomic", body=body))]), max_length=120 + ) + (chunk,) = db.embedded + assert body in chunk.chunk + + +def test_a_manual_section_is_never_split(): + body = " ".join(f"Sentence number {i} says something." for i in range(40)) + db = _ingest(_manifest(manual=[_section(ManualBlock(kind="text", body=body))]), max_length=120) + assert len(db.manual) == 1 and db.manual[0].chunk == body + + +def test_an_unknown_manifest_version_is_refused_before_any_write(tmp_path: pathlib.Path): + path = tmp_path / "corpus.rag.json" + payload = _manifest().model_dump() + payload["version"] = rag_import.SCHEMA_VERSION + 1 + path.write_text(json.dumps(payload)) + with pytest.raises(SystemExit, match="unsupported manifest version"): + rag_import._load_manifest(path) + + +def test_a_valid_manifest_round_trips_through_the_loader(tmp_path: pathlib.Path): + path = tmp_path / "corpus.rag.json" + manifest = _manifest( + manual=[_section(ManualBlock(kind="code", body="x"))], + embedded=[_group(EmbeddedBlock(kind="atomic", body="| a | b |"))], + ) + path.write_text(manifest.model_dump_json()) + loaded = rag_import._load_manifest(path) + assert loaded.manual_sections[0].blocks[0].kind == "code" + assert loaded.embedded_groups[0].blocks[0].kind == "atomic" + + +def test_an_unregistered_knowledge_base_says_where_to_register_it(): + with pytest.raises(SystemExit, match="no connection registered for knowledge_base"): + rag_import._resolve_output(_manifest(kb="no_such_kb"), None) + + +def test_output_overrides_the_registry_lookup(): + # …which is what makes the importer usable before a corpus is registered at all. + conn = "postgresql://elsewhere/rag_db" + assert rag_import._resolve_output(_manifest(kb="no_such_kb"), conn) == conn diff --git a/tests/test_remappings.py b/tests/test_remappings.py index b1af331c..603b4156 100644 --- a/tests/test_remappings.py +++ b/tests/test_remappings.py @@ -17,7 +17,11 @@ from certora_autosetup.build_systems.foundry import FoundryManager from certora_autosetup.utils import remappings as remappings_mod -from certora_autosetup.utils.remappings import build_packages_from_remapping_sources +from certora_autosetup.utils.remappings import ( + build_packages_from_remapping_sources, + node_modules_package_root, + resolve_node_modules_target, +) def _no_forge(monkeypatch: pytest.MonkeyPatch) -> None: @@ -213,3 +217,479 @@ def fake_run(*_args, **kwargs): build_packages_from_remapping_sources(base_dir=tmp_path, log_fn=lambda *_: None, profile="ci") assert captured["env"]["FOUNDRY_PROFILE"] == "ci" + + +def _nested_project(tmp_path: Path) -> Path: + """A repo whose Foundry project sits at /chains/somechain, with a sub-project tree.""" + project = tmp_path / "chains" / "somechain" + (project / "src" / "Widget_1234" / "dependencies" / "oz-5.4.0" / "contracts").mkdir(parents=True) + (project / "lib" / "forge-std" / "src").mkdir(parents=True) + return project + + +def test_context_is_rebased_onto_the_run_root(tmp_path: Path, monkeypatch) -> None: + # `forge remappings` reports contexts relative to the project dir, but solc matches them + # against source unit names, which are relative to the run root. For a project nested under + # the run root the reported context `src/Widget_1234/` never prefixes the source unit name + # `chains/somechain/src/Widget_1234/...`, so the remapping silently never applies and every + # import of that sub-project fails to resolve. + project = _nested_project(tmp_path) + _forge_returning( + monkeypatch, + "src/Widget_1234/:@openzeppelin/contracts/=src/Widget_1234/dependencies/oz-5.4.0/contracts/\n" + "forge-std/=lib/forge-std/src/\n", + ) + + packages = build_packages_from_remapping_sources( + base_dir=project, log_fn=lambda *_: None, run_root=tmp_path + ) + + keys = _keys(packages) + assert "chains/somechain/src/Widget_1234/:@openzeppelin/contracts/" in keys + assert "src/Widget_1234/:@openzeppelin/contracts/" not in keys + # the target half is untouched by the rebasing — still absolute, still pointing at the tree + assert _path_of(packages, "chains/somechain/src/Widget_1234/:@openzeppelin/contracts/") == \ + str(project / "src/Widget_1234/dependencies/oz-5.4.0/contracts") + "/" + # an unscoped key has no context to rebase + assert "forge-std/" in keys + + +def test_context_unchanged_when_the_project_is_the_run_root(tmp_path: Path, monkeypatch) -> None: + # The flat case (the overwhelming majority): base_dir == run_root, so contexts are already + # expressed against the run root and must come out byte-identical. + (tmp_path / "lib" / "some-dependency").mkdir(parents=True) + _no_forge(monkeypatch) + (tmp_path / "remappings.txt").write_text( + "lib/some-dependency/:@openzeppelin/contracts/=lib/openzeppelin-contracts-v4/contracts/\n" + ) + + with_root = build_packages_from_remapping_sources( + base_dir=tmp_path, log_fn=lambda *_: None, run_root=tmp_path + ) + without_root = build_packages_from_remapping_sources(base_dir=tmp_path, log_fn=lambda *_: None) + + assert with_root == without_root + assert "lib/some-dependency/:@openzeppelin/contracts/" in _keys(with_root) + + +def test_context_naming_no_directory_is_left_alone(tmp_path: Path, monkeypatch) -> None: + # Only a context that names a real directory under the project is project-relative. Anything + # else — including a context already written against the run root — is left as authored. + project = _nested_project(tmp_path) + _no_forge(monkeypatch) + (project / "remappings.txt").write_text( + "chains/somechain/src/Widget_1234/:@oz/=src/Widget_1234/dependencies/oz-5.4.0/contracts/\n" + ) + + packages = build_packages_from_remapping_sources( + base_dir=project, log_fn=lambda *_: None, run_root=tmp_path + ) + + assert "chains/somechain/src/Widget_1234/:@oz/" in _keys(packages) + + +def test_context_outside_the_run_root_is_left_alone_with_a_warning(tmp_path: Path, monkeypatch) -> None: + # A context resolving outside the run root cannot be named by any source unit name; keep the + # authored form and say so rather than emitting a `../`-prefixed context. + project = _nested_project(tmp_path) + outside = tmp_path.parent / f"{tmp_path.name}-vendor" + outside.mkdir(exist_ok=True) + _no_forge(monkeypatch) + (project / "remappings.txt").write_text(f"{outside}/:@oz/=lib/forge-std/src/\n") + warnings: list[tuple[str, str]] = [] + + packages = build_packages_from_remapping_sources( + base_dir=project, + log_fn=lambda msg, level: warnings.append((msg, level)), + run_root=tmp_path, + ) + + assert f"{outside}/:@oz/" in _keys(packages) + assert any(level == "WARNING" and "outside the run root" in msg for msg, level in warnings) + + +def test_parse_config_rebases_contexts_against_the_project_root(tmp_path: Path, monkeypatch) -> None: + # End-to-end at the bug site: the manager knows the run root, so parse_config's packages + # come out with run-root-relative contexts. + project = _nested_project(tmp_path) + foundry_toml = project / "foundry.toml" + foundry_toml.write_text( + '[profile.default]\nsrc = "src"\n' + 'remappings = ["src/Widget_1234/:@oz/=src/Widget_1234/dependencies/oz-5.4.0/contracts/"]\n' + ) + _no_forge(monkeypatch) + + manager = FoundryManager(project_root=tmp_path, scope=None) + config = manager.parse_config(foundry_toml) + + keys = {p.split("=", 1)[0] for p in (config.packages or [])} + assert "chains/somechain/src/Widget_1234/:@oz/" in keys + + +def test_context_that_is_the_run_root_is_left_alone_without_a_warning(tmp_path: Path, monkeypatch) -> None: + # A context resolving to the run root itself already covers every source unit name, so + # nothing is wrong with it — unlike a context resolving outside the run root, it must not + # be reported as a problem. + project = _nested_project(tmp_path) + _no_forge(monkeypatch) + (project / "remappings.txt").write_text("../../:@oz/=lib/forge-std/src/\n") + logged: list[tuple[str, str]] = [] + + packages = build_packages_from_remapping_sources( + base_dir=project, log_fn=lambda msg, level: logged.append((msg, level)), run_root=tmp_path + ) + + assert "../../:@oz/" in _keys(packages) + assert not [m for m, level in logged if level == "WARNING" and "run root" in m] + + +# ============================================================================= +# Hoisted node_modules: the ancestor walk +# ============================================================================= +# +# npm/yarn hoist a dependency to the highest node_modules that satisfies every consumer, so a +# sub-project's own node_modules/ frequently does not exist while the repo root's does. +# solc has no such resolver, so the packages list must name the directory that exists. + + +def _hoisted_repo(tmp_path: Path) -> Path: + """A repo whose Foundry project sits at /smart-contracts, with @vault/core hoisted to + the repo root and @widget/lib installed locally in the sub-project.""" + project = tmp_path / "smart-contracts" + (tmp_path / "node_modules" / "@vault" / "core" / "contracts").mkdir(parents=True) + (project / "node_modules" / "@widget" / "lib").mkdir(parents=True) + return project + + +def test_hoisted_package_resolves_from_the_ancestor_while_local_stays_local( + tmp_path: Path, monkeypatch +) -> None: + project = _hoisted_repo(tmp_path) + _no_forge(monkeypatch) + (project / "package.json").write_text( + '{"dependencies": {"@vault/core": "^1.0.0", "@widget/lib": "^1.0.0"}}' + ) + + packages = build_packages_from_remapping_sources( + base_dir=project, log_fn=lambda *_: None, run_root=tmp_path + ) + + assert _path_of(packages, "@vault/core/") == str(tmp_path / "node_modules/@vault/core") + "/" + assert _path_of(packages, "@widget/lib/") == str(project / "node_modules/@widget/lib") + "/" + + +def test_nearest_node_modules_wins_over_the_ancestor(tmp_path: Path, monkeypatch) -> None: + # Node's own resolution order: the closest node_modules answers, even when an ancestor + # also provides the package (routinely a different version). + project = tmp_path / "smart-contracts" + (tmp_path / "node_modules" / "@vault" / "core").mkdir(parents=True) + (project / "node_modules" / "@vault" / "core").mkdir(parents=True) + _no_forge(monkeypatch) + (project / "package.json").write_text('{"dependencies": {"@vault/core": "^1.0.0"}}') + + packages = build_packages_from_remapping_sources( + base_dir=project, log_fn=lambda *_: None, run_root=tmp_path + ) + + assert _path_of(packages, "@vault/core/") == str(project / "node_modules/@vault/core") + "/" + + +def test_walk_stops_at_the_run_root(tmp_path: Path, monkeypatch) -> None: + # certoraRun only uploads the run root's tree, so a package above it is unusable: the walk + # must not reach it, and the base-dir target is emitted instead. + project = tmp_path / "smart-contracts" + project.mkdir() + outside = tmp_path.parent / "node_modules" / "@vault" / "core" + outside.mkdir(parents=True, exist_ok=True) + _no_forge(monkeypatch) + (project / "package.json").write_text('{"dependencies": {"@vault/core": "^1.0.0"}}') + + packages = build_packages_from_remapping_sources( + base_dir=project, log_fn=lambda *_: None, run_root=tmp_path + ) + + assert _path_of(packages, "@vault/core/") == str(project / "node_modules/@vault/core") + "/" + + +def test_no_run_root_performs_no_walk(tmp_path: Path, monkeypatch) -> None: + # Every caller that does not know the run root keeps the one-candidate behaviour. + project = _hoisted_repo(tmp_path) + _no_forge(monkeypatch) + (project / "package.json").write_text('{"dependencies": {"@vault/core": "^1.0.0"}}') + + packages = build_packages_from_remapping_sources(base_dir=project, log_fn=lambda *_: None) + + assert _path_of(packages, "@vault/core/") == str(project / "node_modules/@vault/core") + "/" + + +def test_flat_project_packages_are_identical_with_and_without_run_root( + tmp_path: Path, monkeypatch +) -> None: + # The overwhelming majority of projects: base_dir == run_root, so the walk has exactly one + # candidate and the packages list must come out byte-identical to the no-run-root result. + (tmp_path / "node_modules" / "@vault" / "core").mkdir(parents=True) + (tmp_path / "lib" / "widget").mkdir(parents=True) + _no_forge(monkeypatch) + (tmp_path / "remappings.txt").write_text( + "@vault/core/=node_modules/@vault/core/\n" + "widget/=lib/widget/\n" + "@absent/pkg/=node_modules/@absent/pkg/\n" + ) + + with_root = build_packages_from_remapping_sources( + base_dir=tmp_path, log_fn=lambda *_: None, run_root=tmp_path + ) + without_root = build_packages_from_remapping_sources(base_dir=tmp_path, log_fn=lambda *_: None) + + assert with_root == without_root + assert _path_of(with_root, "@vault/core/") == str(tmp_path / "node_modules/@vault/core") + "/" + + +def test_existing_local_target_is_never_rewritten(tmp_path: Path, monkeypatch) -> None: + # The walk can only change an entry whose target does not exist; a local install always wins. + project = tmp_path / "smart-contracts" + (project / "node_modules" / "@vault" / "core" / "contracts").mkdir(parents=True) + (tmp_path / "node_modules" / "@vault" / "core" / "contracts").mkdir(parents=True) + _no_forge(monkeypatch) + (project / "remappings.txt").write_text("@vault/=node_modules/@vault/core/contracts/\n") + + packages = build_packages_from_remapping_sources( + base_dir=project, log_fn=lambda *_: None, run_root=tmp_path + ) + + assert _path_of(packages, "@vault/") == \ + str(project / "node_modules/@vault/core/contracts") + "/" + + +def test_hoist_resolution_is_independent_of_the_entry_source(tmp_path: Path, monkeypatch) -> None: + # The resolution is a property of the target, not of which file the entry came from. + project = _hoisted_repo(tmp_path) + expected = str(tmp_path / "node_modules/@vault/core") + "/" + _no_forge(monkeypatch) + + (project / "remappings.txt").write_text("@vault/core/=node_modules/@vault/core/\n") + from_remappings_txt = build_packages_from_remapping_sources( + base_dir=project, log_fn=lambda *_: None, run_root=tmp_path + ) + (project / "remappings.txt").unlink() + (project / "foundry.toml").write_text( + '[profile.default]\nremappings = ["@vault/core/=node_modules/@vault/core/"]\n' + ) + from_foundry_toml = build_packages_from_remapping_sources( + base_dir=project, log_fn=lambda *_: None, run_root=tmp_path + ) + + assert _path_of(from_remappings_txt, "@vault/core/") == expected + assert _path_of(from_foundry_toml, "@vault/core/") == expected + + +def test_lib_target_is_never_walked(tmp_path: Path, monkeypatch) -> None: + # forge and soldeer do not hoist, and a sibling project's lib/ is routinely a + # different pin — walking those would silently bind the wrong version. + project = tmp_path / "smart-contracts" + project.mkdir() + (tmp_path / "lib" / "oz").mkdir(parents=True) + _no_forge(monkeypatch) + (project / "remappings.txt").write_text("@oz/=lib/oz/\n") + + packages = build_packages_from_remapping_sources( + base_dir=project, log_fn=lambda *_: None, run_root=tmp_path + ) + + assert _path_of(packages, "@oz/") == str(project / "lib/oz") + "/" + + +def test_unscoped_package_name_is_one_segment(tmp_path: Path, monkeypatch) -> None: + # A scoped name spans two segments (@scope/name), an unscoped one exactly one — splitting + # wrongly would test the existence of the wrong directory. + project = tmp_path / "smart-contracts" + project.mkdir() + (tmp_path / "node_modules" / "plainpkg" / "src").mkdir(parents=True) + _no_forge(monkeypatch) + (project / "remappings.txt").write_text("plainpkg/=node_modules/plainpkg/src/\n") + + packages = build_packages_from_remapping_sources( + base_dir=project, log_fn=lambda *_: None, run_root=tmp_path + ) + + assert _path_of(packages, "plainpkg/") == str(tmp_path / "node_modules/plainpkg/src") + "/" + + +def test_subpath_inside_a_hoisted_package_resolves(tmp_path: Path, monkeypatch) -> None: + project = tmp_path / "smart-contracts" + project.mkdir() + (tmp_path / "node_modules" / "@pkg" / "artifacts" / "src").mkdir(parents=True) + _no_forge(monkeypatch) + (project / "foundry.toml").write_text( + '[profile.default]\nremappings = ["@pkg/=node_modules/@pkg/artifacts/src/"]\n' + ) + + packages = build_packages_from_remapping_sources( + base_dir=project, log_fn=lambda *_: None, run_root=tmp_path + ) + + assert _path_of(packages, "@pkg/") == str(tmp_path / "node_modules/@pkg/artifacts/src") + "/" + + +def test_ancestor_with_the_subpath_beats_a_nearer_package_without_it( + tmp_path: Path, monkeypatch +) -> None: + # Only the full target is usable by solc, so a nearer package that lacks the remapped + # subdirectory is skipped in favour of an ancestor that has the whole thing. + project = tmp_path / "smart-contracts" + # The nearer install really is the package directory (`@pkg/artifacts`, two segments for a + # scoped name) — it just lacks the remapped `src/`. + (project / "node_modules" / "@pkg" / "artifacts").mkdir(parents=True) + (tmp_path / "node_modules" / "@pkg" / "artifacts" / "src").mkdir(parents=True) + _no_forge(monkeypatch) + (project / "remappings.txt").write_text("@pkg/=node_modules/@pkg/artifacts/src/\n") + logged: list[tuple[str, str]] = [] + + packages = build_packages_from_remapping_sources( + base_dir=project, + log_fn=lambda msg, level: logged.append((msg, level)), + run_root=tmp_path, + ) + + assert _path_of(packages, "@pkg/") == str(tmp_path / "node_modules/@pkg/artifacts/src") + "/" + assert not [m for m, level in logged if level == "WARNING"] + assert any(level == "INFO" and "hoisted install" in m for m, level in logged) + + +def test_installed_package_without_the_subpath_is_reported_as_subpath_missing( + tmp_path: Path, monkeypatch +) -> None: + # No ancestor has the full target either, so the installed package is the best evidence + # there is: the entry keeps naming its subdirectory and the log says the package is there + # while the remapped subdirectory is not — a different remedy from "not installed". + project = tmp_path / "smart-contracts" + (project / "node_modules" / "@oz" / "contracts").mkdir(parents=True) + _no_forge(monkeypatch) + (project / "remappings.txt").write_text("@oz/=node_modules/@oz/contracts/token/\n") + logged: list[tuple[str, str]] = [] + + resolution = resolve_node_modules_target( + "node_modules/@oz/contracts/token", base_dir=project, run_root=tmp_path + ) + packages = build_packages_from_remapping_sources( + base_dir=project, + log_fn=lambda msg, level: logged.append((msg, level)), + run_root=tmp_path, + ) + + assert resolution.kind == "subpath_missing" + assert resolution.package_dir == str(project / "node_modules/@oz/contracts") + assert _path_of(packages, "@oz/") == str(project / "node_modules/@oz/contracts/token") + "/" + warnings = [m for m, level in logged if level == "WARNING"] + assert any("exists but the remapped subdirectory is missing" in m for m in warnings) + + +def test_ancestor_walk_never_leaves_the_run_root_through_a_symlinked_base_dir( + tmp_path: Path, +) -> None: + # A base_dir that reaches the run root through a symlink must not widen the walk: a package + # found above the run root is outside the tree certoraRun uploads. + run_root = tmp_path / "repo" + (run_root / "a" / "b" / "c").mkdir(parents=True) + (tmp_path / "node_modules" / "@vault" / "core").mkdir(parents=True) + base_dir = run_root / "link" + base_dir.symlink_to(run_root / "a" / "b" / "c", target_is_directory=True) + + resolution = resolve_node_modules_target( + "node_modules/@vault/core", base_dir=base_dir, run_root=run_root + ) + + assert resolution.kind == "unresolved" + assert all(str(tmp_path / "node_modules") not in candidate for candidate in resolution.searched) + + +def test_ancestor_walk_reaches_the_run_root_through_a_symlinked_base_dir(tmp_path: Path) -> None: + # The converse: a base_dir textually deeper than it resolves must still be walked all the + # way up to the run root, or a hoisted package there is silently missed. + run_root = tmp_path / "repo" + (run_root / "x").mkdir(parents=True) + (run_root / "p" / "q").mkdir(parents=True) + hoisted = run_root / "node_modules" / "@vault" / "core" + hoisted.mkdir(parents=True) + base_dir = run_root / "p" / "q" / "s" + base_dir.symlink_to(run_root / "x", target_is_directory=True) + + resolution = resolve_node_modules_target( + "node_modules/@vault/core", base_dir=base_dir, run_root=run_root + ) + + assert resolution.kind == "hoisted" + assert resolution.path == str(hoisted) + + +def test_package_root_of_a_target_takes_the_last_node_modules(tmp_path: Path) -> None: + # Scoped names span two segments, unscoped one, and a nested install is governed by the + # innermost node_modules — the classifier tests exactly this directory for existence. + assert node_modules_package_root("/r/node_modules/@oz/contracts/token") == \ + "/r/node_modules/@oz/contracts" + assert node_modules_package_root("/r/node_modules/solady/src") == "/r/node_modules/solady" + assert node_modules_package_root("/r/node_modules/a/node_modules/@b/c/src") == \ + "/r/node_modules/a/node_modules/@b/c" + assert node_modules_package_root("/r/node_modules/@oz/contracts") == \ + "/r/node_modules/@oz/contracts" + assert node_modules_package_root("/r/lib/openzeppelin/contracts") is None + + +def test_package_missing_everywhere_keeps_the_base_dir_target_and_warns( + tmp_path: Path, monkeypatch +) -> None: + # Dropping the entry would turn a precise `Source "…" not found` into a vaguer failure, or + # let the bare import resolve against a same-named path in the project tree. + project = tmp_path / "smart-contracts" + project.mkdir() + _no_forge(monkeypatch) + (project / "remappings.txt").write_text("@vault/=node_modules/@vault/core/\n") + logged: list[tuple[str, str]] = [] + + packages = build_packages_from_remapping_sources( + base_dir=project, + log_fn=lambda msg, level: logged.append((msg, level)), + run_root=tmp_path, + ) + + assert _path_of(packages, "@vault/") == str(project / "node_modules/@vault/core") + "/" + warnings = [m for m, level in logged if level == "WARNING" and "does not exist" in m] + assert warnings + assert str(tmp_path / "node_modules/@vault/core") in warnings[0] + + +def test_prefixed_node_modules_target_is_left_untouched(tmp_path: Path, monkeypatch) -> None: + # An explicit prefix before node_modules names a location the author chose; only a bare + # node_modules/... target is the node-resolution idiom hoisting applies to. + project = tmp_path / "smart-contracts" + project.mkdir() + (tmp_path / "node_modules" / "@vault" / "core").mkdir(parents=True) + _no_forge(monkeypatch) + (project / "remappings.txt").write_text("@vault/=packages/a/node_modules/@vault/core/\n") + + packages = build_packages_from_remapping_sources( + base_dir=project, log_fn=lambda *_: None, run_root=tmp_path + ) + + assert _path_of(packages, "@vault/") == \ + str(project / "packages/a/node_modules/@vault/core") + "/" + + +def test_parse_config_emits_run_root_relative_hoisted_packages(tmp_path: Path, monkeypatch) -> None: + # End-to-end: a hoisted target stays inside the run root, so the conf keeps relative paths + # (BuildSystemConfig._relativize_packages does a textual relative_to against the run root). + project = _hoisted_repo(tmp_path) + foundry_toml = project / "foundry.toml" + foundry_toml.write_text( + '[profile.default]\nsrc = "src"\nremappings = ["@vault/=node_modules/@vault/core/contracts/"]\n' + ) + _no_forge(monkeypatch) + # _relativize_packages relativizes against the process CWD, which in a run IS the run root. + monkeypatch.chdir(tmp_path) + + manager = FoundryManager(project_root=project, scope=None, run_root=tmp_path) + config = manager.parse_config(foundry_toml) + packages = config.to_certora_dict()["packages"] + + # Relative (no `../`, no absolute fallback): the walk never leaves the run root. + assert packages == ["@vault/=node_modules/@vault/core/contracts"] diff --git a/tests/test_rule_skips.py b/tests/test_rule_skips.py index 4bc040c3..1ad3b4b7 100644 --- a/tests/test_rule_skips.py +++ b/tests/test_rule_skips.py @@ -10,8 +10,10 @@ from composer.spec.source.prover import ( StateWithSkips, VALIDATION_KEY, ) -from composer.spec.cvl_generation import check_completion +from composer.authoring.state import check_completion from composer.prover.core import ProverReport +from composer.prover.ptypes import RulePath +from composer.prover.results import StatusCodes from graphcore.testing import Scenario, tool_call_raw, ToolCallDict from graphcore.tools.results import result_tool_generator @@ -61,13 +63,27 @@ def _result(commentary: str) -> ToolCallDict: # --------------------------------------------------------------------------- +def _spec_decls(*rules: str) -> str: + """A spec declaring exactly ``rules`` — the mocked ``declared_rules_list`` parses + these declarations back out, so a report's rules must be declared here for the + completion check to treat them as the spec's.""" + return "\n".join(f"rule {r} {{ assert true; }}" for r in rules) + + def _raw_report(**rule_status: bool) -> ProverReport: - return ProverReport(rule_status=rule_status, result_str="Prover report output", link="local://test-run") + return ProverReport(result_str="Prover report output", link="local://test-run", raw_rule_status={ + RulePath(rule=k): "VERIFIED" if v else "VIOLATED" for (k,v) in rule_status.items() + }, + certora_run_stdout="certoraRun output" + ) def _summarized_report(todo: str, **rule_status: bool) -> ProverReport: return ProverReport( - rule_status=rule_status, result_str=todo, link="local://test-run", + result_str=todo, link="local://test-run", raw_rule_status={ + RulePath(rule=k): "VERIFIED" if v else "VIOLATED" for (k,v) in rule_status.items() + }, + certora_run_stdout="certoraRun output" ) @@ -106,6 +122,7 @@ def _scenario( required_validations=required if required is not None else [VALIDATION_KEY], rule_skips=rule_skips or {}, config={"files": ["src/Foo.sol"]}, + reminders_channel=[], # verify_spec's stamp is bound to the applied-edit history; the source # pipeline always seeds it, so the test state must too. version_history=[], @@ -158,6 +175,7 @@ async def test_summarized_report_returns_todo(self, certora_prover: ProverMock): msg = await _scenario( certora_prover, _summarized_report("1. Fix rule foo\n2. Fix rule bar", foo=False, bar=False), + curr_spec=_spec_decls("foo", "bar"), ).turn( _verify() ).run_last_single_tool(_PROVER) @@ -167,6 +185,7 @@ async def test_raw_report_failures_no_stamp(self, certora_prover: ProverMock): assert await _scenario( certora_prover, _raw_report(foo=True, bar=False), + curr_spec=_spec_decls("foo", "bar"), ).turns( _verify(), _result("done"), @@ -176,15 +195,19 @@ async def test_raw_report_all_verified_stamps(self, certora_prover: ProverMock): assert await _scenario( certora_prover, _raw_report(foo=True, bar=True), + curr_spec=_spec_decls("foo", "bar"), ).turns( _verify(), _result("done"), ).map_run(_result_accepted) == "done" - async def test_filtered_rules_dont_stamp(self, certora_prover: ProverMock): + async def test_partial_coverage_doesnt_stamp(self, certora_prover: ProverMock): + # A rule-scoped run that verifies only part of the declared rules must not + # stamp — bar was never exercised against this spec. assert await _scenario( certora_prover, _raw_report(foo=True), + curr_spec=_spec_decls("foo", "bar"), ).turns( _verify_rules("foo"), _result("done"), @@ -202,6 +225,7 @@ async def test_skipped_failure_counts_as_verified(self, certora_prover: ProverMo assert await _scenario( certora_prover, _raw_report(ruleA=False, ruleB=True), + curr_spec=_spec_decls("ruleA", "ruleB"), ).turn( _skip("ruleA", "known issue"), ).turns( @@ -214,6 +238,7 @@ async def test_unskipped_failure_blocks_verification(self, certora_prover: Prove assert await _scenario( certora_prover, _raw_report(ruleA=False, ruleB=True), + curr_spec=_spec_decls("ruleA", "ruleB"), ).turn( _skip("ruleA", "temp"), ).turn( @@ -228,6 +253,7 @@ async def test_non_skipped_failure_blocks_despite_other_skips(self, certora_prov assert await _scenario( certora_prover, _raw_report(ruleA=False, ruleB=False), + curr_spec=_spec_decls("ruleA", "ruleB"), ).turn( _skip("ruleA", "known"), ).turns( diff --git a/tests/test_rules_striping.py b/tests/test_rules_striping.py new file mode 100644 index 00000000..bea6cc8b --- /dev/null +++ b/tests/test_rules_striping.py @@ -0,0 +1,531 @@ +"""Tests for rules striping: satisfying a spec's rules piecemeal across several +``verify_spec`` calls (rule-scoped includes and excludes) instead of one full run. + +Covers: + +- the pure helpers — which rules a logged run executed (``_executed_rules``) and + whether the run history against the current authoring state adds up to full + coverage (``_is_completion_history``); +- the ``verify_spec`` tool surface — include/exclude plumbing into the conf, the + ``ProverRunLog`` entries, and completion (validation stamp + reminder) arriving + on whichever run completes coverage, however it was scoped; +- ``declared_rules_list``, with its certoraRun/typechecker subprocesses faked; +- the ``known_rules`` cross-check of ``validate_property_rules``. + +The prover core is mocked throughout (the ``certora_prover`` fixture's seams: +``run_prover`` + ``declared_rules_list``); no prover jobs run. +""" +import json +from pathlib import Path +from typing import Annotated + +import pytest + +from langchain_core.tools import InjectedToolCallId, tool +from langgraph.types import Command + +from composer.authoring.state import check_completion, spec_digest +from composer.prover.core import ProverReport, declared_rules_list +from composer.prover.ptypes import RulePath, StatusCodes +from composer.spec.cvl_generation import PropertyRuleMapping, validate_property_rules +from composer.spec.source.author import ExpectRuleFailure +from composer.spec.source.prover import ( + NagMarker, ProverHistoryItem, ProverRunLog, RuleSelection, StateWithSkips, + VALIDATION_KEY, _executed_rules, _is_completion_history, +) +from composer.spec.types import PropertyTitle, RuleName + +from graphcore.graph import tool_state_update +from graphcore.testing import Scenario, ToolCallDict, tool_call_raw +from graphcore.tools.results import result_tool_generator + +from .conftest import ProverMock, ProverToolResponse + +RA = RulePath(rule="a") +RB = RulePath(rule="b") + + +# --------------------------------------------------------------------------- +# ProverRunLog constructors +# --------------------------------------------------------------------------- + + +def _inc(*rules: str) -> RuleSelection: + return {"sort": "include", "selector": list(rules)} + + +def _exc(*rules: str) -> RuleSelection: + return {"sort": "exclude", "selector": list(rules)} + + +def _log( + *results: tuple[RulePath, StatusCodes], + digest: str = "d1", + rules: RuleSelection | None = None, + declared: tuple[str, ...] = ("a", "b"), +) -> ProverRunLog: + return ProverRunLog( + tool_call_id="tc", + prover_results=list(results), + rules=rules, + spec_digest="spec-hash", + sort="run", + declared_rules=list(declared), + state_digest=digest, + ) + + +# ========================================================================= +# _executed_rules: which rules a logged run actually exercised +# ========================================================================= + + +class TestExecutedRules: + def test_full_run_executes_all_declared(self): + assert _executed_rules(_log(declared=("a", "b", "c"))) == ["a", "b", "c"] + + def test_include_executes_the_selector(self): + assert _executed_rules(_log(rules=_inc("b"), declared=("a", "b", "c"))) == ["b"] + + def test_exclude_executes_the_complement(self): + assert _executed_rules(_log(rules=_exc("b"), declared=("a", "b", "c"))) == ["a", "c"] + + +# ========================================================================= +# _is_completion_history: piecemeal coverage accounting +# ========================================================================= + + +def _complete( + history: list[ProverHistoryItem], + curr: list[tuple[RulePath, StatusCodes]], + *, + digest: str = "d1", + expected_to_fail: set[str] | None = None, + all_rules: tuple[str, ...] = ("a", "b"), +) -> bool: + return _is_completion_history( + l=history, + curr_digest=digest, + expected_to_fail=expected_to_fail or set(), + curr_status=curr, + all_rules=list(all_rules), + ) + + +class TestCompletionHistory: + def test_single_full_run_completes(self): + assert _complete([], [(RA, "VERIFIED"), (RB, "VERIFIED")]) + + def test_piecemeal_runs_complete_together(self): + history: list[ProverHistoryItem] = [_log((RA, "VERIFIED"), rules=_inc("a"))] + assert _complete(history, [(RB, "VERIFIED")]) + + def test_uncovered_rule_blocks(self): + assert not _complete([], [(RA, "VERIFIED")]) + + def test_current_failure_blocks(self): + assert not _complete([], [(RA, "VERIFIED"), (RB, "VIOLATED")]) + + def test_expected_failure_is_forgiven_and_covered(self): + assert _complete( + [], [(RA, "VERIFIED"), (RB, "VIOLATED")], expected_to_fail={"b"} + ) + + def test_historic_failure_of_unskipped_rule_blocks(self): + history: list[ProverHistoryItem] = [_log((RB, "TIMEOUT"), rules=_inc("b"))] + assert not _complete(history, [(RA, "VERIFIED")]) + + def test_state_digest_mismatch_severs_coverage(self): + history: list[ProverHistoryItem] = [_log((RA, "VERIFIED"), digest="d0")] + assert not _complete(history, [(RB, "VERIFIED")], digest="d1") + + def test_stale_run_stops_the_walk(self): + # The walk stops at the first run against a different state: coverage from a + # matching run BEHIND it does not count, even though its digest matches. + history: list[ProverHistoryItem] = [ + _log((RA, "VERIFIED"), digest="d1"), + _log((RB, "VERIFIED"), digest="d0"), + ] + assert not _complete(history, [(RB, "VERIFIED")], digest="d1") + + def test_nag_markers_are_transparent(self): + history: list[ProverHistoryItem] = [ + _log((RA, "VERIFIED"), rules=_inc("a")), + NagMarker(sort="nag", nagged_rules=[RA]), + ] + assert _complete(history, [(RB, "VERIFIED")]) + + def test_overlapping_coverage_completes(self): + # A rule re-verified by the current run also appears in the matching history + # entry that supplies the rest of the coverage. + history: list[ProverHistoryItem] = [_log((RA, "VERIFIED"), (RB, "VERIFIED"))] + assert _complete(history, [(RA, "VERIFIED")]) + + def test_overlapping_coverage_that_stays_incomplete_terminates(self): + # Same rule verified twice with nothing covering the rest: must simply report + # incomplete (this is the shape that used to re-walk the same history entry). + history: list[ProverHistoryItem] = [_log((RA, "VERIFIED"), rules=_inc("a"))] + assert not _complete(history, [(RA, "VERIFIED")]) + + def test_undeclared_result_rules_are_ignored(self): + # The prover reports checks the declared-rules list withholds (e.g. the + # envfree static check); they must count for nothing rather than crash. + static_check = RulePath(rule="envfreeFuncsStaticCheck") + assert _complete( + [], [(RA, "VERIFIED"), (static_check, "VERIFIED")], all_rules=("a",) + ) + + def test_parametric_instantiations_share_one_rule(self): + assert _complete( + [], + [ + (RulePath(rule="a", method="f()"), "VERIFIED"), + (RulePath(rule="a", method="g()"), "VERIFIED"), + (RB, "VERIFIED"), + ], + ) + + +# ========================================================================= +# declared_rules_list: the certoraRun/typechecker rule-listing pre-pass +# ========================================================================= + + +class _FakeProc: + def __init__(self, rc: int): + self._rc = rc + + async def wait(self) -> int: + return self._rc + + +def _install_fake_prover_procs( + monkeypatch, + *, + certora_rc: int = 0, + java_rc: int = 0, + rules_text: str = "a\nb\n", + conf_msg: str | None = None, +): + """Fake the two subprocesses of ``declared_rules_list``: certoraRun materializes a + build-mirror dir whose run.conf carries the ``--msg`` key it was passed (or + ``conf_msg``, to simulate a foreign run), java writes ``rules_text`` to its + ``-listRules`` target.""" + + async def fake_exec(*argv, cwd=None, stdout=None, stderr=None): + assert cwd is not None + argv = [str(a) for a in argv] + if argv[0] == "certoraRun": + assert "--compilation_steps_only" in argv + key = argv[argv.index("--msg") + 1] + build = Path(cwd) / ".certora_internal" / "build_mirror" + build.mkdir(parents=True, exist_ok=True) + (build / "run.conf").write_text( + json.dumps({"msg": conf_msg if conf_msg is not None else key}) + ) + return _FakeProc(certora_rc) + assert argv[0] == "java" + Path(argv[argv.index("-listRules") + 1]).write_text(rules_text) + return _FakeProc(java_rc) + + monkeypatch.setattr("asyncio.subprocess.create_subprocess_exec", fake_exec) + + +@pytest.mark.asyncio +class TestDeclaredRulesList: + async def test_lists_rules_filtering_the_static_check(self, tmp_path, monkeypatch): + _install_fake_prover_procs( + monkeypatch, rules_text="a\n b \n\nenvfreeFuncsStaticCheck\nc\n" + ) + assert await declared_rules_list(tmp_path, ["x.conf"]) == ["a", "b", "c"] + + async def test_rejects_caller_supplied_msg(self, tmp_path): + with pytest.raises(ValueError, match="msg"): + await declared_rules_list(tmp_path, ["x.conf", "--msg", "hello"]) + + async def test_build_failure_raises(self, tmp_path, monkeypatch): + _install_fake_prover_procs(monkeypatch, certora_rc=1) + with pytest.raises(ValueError): + await declared_rules_list(tmp_path, ["x.conf"]) + + async def test_typechecker_failure_raises(self, tmp_path, monkeypatch): + _install_fake_prover_procs(monkeypatch, java_rc=1) + with pytest.raises(ValueError): + await declared_rules_list(tmp_path, ["x.conf"]) + + async def test_unmatched_build_dir_raises(self, tmp_path, monkeypatch): + _install_fake_prover_procs(monkeypatch, conf_msg="someone else's run") + with pytest.raises(ValueError, match="build dir"): + await declared_rules_list(tmp_path, ["x.conf"]) + + async def test_discovery_ignores_decoy_entries(self, tmp_path, monkeypatch): + # Pre-existing .certora_internal clutter: a plain file, a dir without a + # run.conf, one with unparseable json, one with a non-string msg, one with + # another run's msg. Discovery must land on the dir the fake writes. + internal = tmp_path / ".certora_internal" + (internal / "no_conf_dir").mkdir(parents=True) + (internal / "plain_file").write_text("not a dir") + bad_json = internal / "bad_json" + bad_json.mkdir() + (bad_json / "run.conf").write_text("{oops") + bad_msg = internal / "bad_msg" + bad_msg.mkdir() + (bad_msg / "run.conf").write_text(json.dumps({"msg": 42})) + other = internal / "other_run" + other.mkdir() + (other / "run.conf").write_text(json.dumps({"msg": "not this one"})) + + _install_fake_prover_procs(monkeypatch, rules_text="a\n") + assert await declared_rules_list(tmp_path, ["x.conf"]) == ["a"] + + +# ========================================================================= +# verify_spec: striped runs through the tool surface +# ========================================================================= + +_PROVER = "verify_spec" +_SKIP = "expect_rule_failure" +_RESULT = "result" + + +result_tool = result_tool_generator( + "result", + (str, "Commentary"), + "Signal completion", + validator=(StateWithSkips, lambda st, *_: check_completion(st)), +) + + +@tool +def set_spec( + spec: str, + tool_call_id: Annotated[str, InjectedToolCallId], +) -> Command: + """Replace the spec under authoring (a digest-changing edit).""" + return tool_state_update(tool_call_id=tool_call_id, content="spec updated", curr_spec=spec) + + +def _spec_decls(*rules: str) -> str: + """A spec declaring exactly ``rules`` — the mocked ``declared_rules_list`` + parses these declarations back out as the run's declared-rules ground truth.""" + return "\n".join(f"rule {r} {{ assert true; }}" for r in rules) + + +def _report(**rule_status: bool) -> ProverReport: + return ProverReport( + result_str="Prover report output", + link="local://test-run", + raw_rule_status={ + RulePath(rule=k): "VERIFIED" if v else "VIOLATED" + for (k, v) in rule_status.items() + }, + certora_run_stdout="" + ) + + +def _verify( + rules: list[str] | None = None, exclude_rules: list[str] | None = None +) -> ToolCallDict: + return tool_call_raw(_PROVER, rules=rules, exclude_rules=exclude_rules) + + +def _result(commentary: str) -> ToolCallDict: + return tool_call_raw(_RESULT, value=commentary) + + +def _set_spec(spec: str) -> ToolCallDict: + return tool_call_raw("set_spec", spec=spec) + + +def _skip(rule_name: str, reason: str) -> ToolCallDict: + return tool_call_raw(_SKIP, rule_name=rule_name, reason=reason) + + +def _scenario( + certora_prover: ProverMock, + *responses: ProverToolResponse, + curr_spec: str, + rule_skips: dict[str, str] | None = None, +): + tools = [ + certora_prover(responses), + ExpectRuleFailure.as_tool(_SKIP), + result_tool, + set_spec, + ] + return Scenario(StateWithSkips, *tools).init( + curr_spec=curr_spec, + skipped=[], + property_rules=[], + validations={}, + required_validations=[VALIDATION_KEY], + rule_skips=rule_skips or {}, + config={"files": ["src/Foo.sol"]}, + reminders_channel=[], + version_history=[], + ) + + +def _result_accepted(st: StateWithSkips) -> str: + assert "result" in st + return st["result"] + + +def _is_result_rejection(st: StateWithSkips) -> bool: + return "result" not in st and Scenario.last_single_tool( + _RESULT, st + ).startswith("Completion REJECTED:") + + +@pytest.mark.asyncio +class TestStripedVerification: + async def test_rules_and_exclude_rules_mutually_exclusive(self, certora_prover: ProverMock): + msg = await _scenario( + certora_prover, curr_spec=_spec_decls("a", "b"), + ).turn( + _verify(rules=["a"], exclude_rules=["b"]) + ).run_last_single_tool(_PROVER) + assert "both" in msg + assert certora_prover.calls == [] + + async def test_include_selection_reaches_conf_and_history(self, certora_prover: ProverMock): + spec = _spec_decls("a", "b") + history = await _scenario( + certora_prover, _report(a=True), curr_spec=spec, + ).turn( + _verify(rules=["a"]) + ).map_run(lambda st: st["prover_history"]) + [entry] = history + assert entry["sort"] == "run" + assert entry["rules"] == {"sort": "include", "selector": ["a"]} + assert entry["declared_rules"] == ["a", "b"] + assert entry["state_digest"] == spec_digest(spec, [], []) + [call] = certora_prover.calls + assert call.conf["rule"] == ["a"] + assert "exclude_rule" not in call.conf + + async def test_exclude_selection_reaches_conf_and_history(self, certora_prover: ProverMock): + history = await _scenario( + certora_prover, _report(a=True), curr_spec=_spec_decls("a", "b"), + ).turn( + _verify(exclude_rules=["b"]) + ).map_run(lambda st: st["prover_history"]) + [entry] = history + assert entry["sort"] == "run" + assert entry["rules"] == {"sort": "exclude", "selector": ["b"]} + [call] = certora_prover.calls + assert call.conf["exclude_rule"] == ["b"] + assert "rule" not in call.conf + + async def test_piecemeal_completion_stamps(self, certora_prover: ProverMock): + # The whole point of striping: two rule-scoped runs that together cover the + # spec complete the task, no full run required. + assert await _scenario( + certora_prover, _report(a=True), _report(b=True), + curr_spec=_spec_decls("a", "b"), + ).turns( + _verify(rules=["a"]), + _verify(rules=["b"]), + _result("done"), + ).map_run(_result_accepted) == "done" + + async def test_exclude_run_alone_doesnt_complete(self, certora_prover: ProverMock): + assert await _scenario( + certora_prover, _report(a=True), curr_spec=_spec_decls("a", "b"), + ).turns( + _verify(exclude_rules=["b"]), + _result("done"), + ).map_run(_is_result_rejection) + + async def test_exclude_then_include_completes(self, certora_prover: ProverMock): + assert await _scenario( + certora_prover, _report(a=True, b=True), _report(c=True), + curr_spec=_spec_decls("a", "b", "c"), + ).turns( + _verify(exclude_rules=["c"]), + _verify(rules=["c"]), + _result("done"), + ).map_run(_result_accepted) == "done" + + async def test_spec_edit_resets_piecemeal_coverage(self, certora_prover: ProverMock): + spec_v1 = _spec_decls("a", "b") + spec_v2 = spec_v1 + "\n// tightened" + assert await _scenario( + certora_prover, _report(a=True), _report(b=True), curr_spec=spec_v1, + ).turns( + _verify(rules=["a"]), + _set_spec(spec_v2), + _verify(rules=["b"]), + _result("done"), + ).map_run(_is_result_rejection) + + async def test_skipped_rule_failure_counts_toward_coverage(self, certora_prover: ProverMock): + assert await _scenario( + certora_prover, _report(a=True), _report(b=False), + curr_spec=_spec_decls("a", "b"), + ).turn( + _skip("b", "known limitation"), + ).turns( + _verify(rules=["a"]), + _verify(rules=["b"]), + _result("done"), + ).map_run(_result_accepted) == "done" + + async def test_completion_reminder_delivered_on_completing_run(self, certora_prover: ProverMock): + reminders = await _scenario( + certora_prover, _report(a=True), _report(b=True), + curr_spec=_spec_decls("a", "b"), + ).turns( + _verify(rules=["a"]), + _verify(rules=["b"]), + ).map_run(lambda st: st["reminders_channel"]) + assert any("task is completed" in r for r in reminders) + + async def test_no_completion_reminder_while_coverage_is_partial(self, certora_prover: ProverMock): + reminders = await _scenario( + certora_prover, _report(a=True), curr_spec=_spec_decls("a", "b"), + ).turn( + _verify(rules=["a"]), + ).map_run(lambda st: st["reminders_channel"]) + assert reminders == [] + + +# ========================================================================= +# validate_property_rules: the known_rules cross-check +# ========================================================================= + + +def _mapping(title: str, *rules: str) -> PropertyRuleMapping: + return PropertyRuleMapping( + property_title=PropertyTitle(title), rules=[RuleName(r) for r in rules] + ) + + +class TestValidatePropertyRules: + def test_known_rules_reject_unran_claims(self): + err = validate_property_rules( + [_mapping("p1", "ghost_rule")], [], [PropertyTitle("p1")], + known_rules={"real_rule"}, + ) + assert err is not None and "ghost_rule" in err + + def test_known_rules_reject_unclaimed_rules(self): + err = validate_property_rules( + [_mapping("p1", "a")], [], [PropertyTitle("p1")], + known_rules={"a", "orphan"}, + ) + assert err is not None and "orphan" in err + + def test_matching_known_rules_accepted(self): + assert validate_property_rules( + [_mapping("p1", "a"), _mapping("p2", "b")], + [], + [PropertyTitle("p1"), PropertyTitle("p2")], + known_rules={"a", "b"}, + ) is None + + def test_without_known_rules_names_arent_cross_checked(self): + assert validate_property_rules( + [_mapping("p1", "anything_goes")], [], [PropertyTitle("p1")], + ) is None diff --git a/tests/test_rust_frontend.py b/tests/test_rust_frontend.py new file mode 100644 index 00000000..ac784993 --- /dev/null +++ b/tests/test_rust_frontend.py @@ -0,0 +1,86 @@ +"""Generic Rust frontend event routing (no wheel / no running TUI). + +A declared ``notice`` event kind (e.g. Crucible's per-invariant ``verdict``) must be +surfaced as a persistent callout via ``post_notice`` — not buried in the collapsible +events log — while ordinary kinds still stream to the log and undeclared kinds are +ignored. The notice headline carries an outcome glyph (✓/✗) when the payload has one. +""" + +import asyncio +from typing import Any, cast + +from composer.rustapp.frontend import GenericRustTaskHandler, _notice_headline +from composer.spec.source.report.render import outcome_glyph +from composer.spec.source.report.schema import Outcome +from composer.ui.tool_display import ToolDisplayConfig + + +def test_notice_headline_prefixes_outcome_glyph(): + assert _notice_headline({"outcome": "GOOD", "line": "held"}) == "✓ held" + assert _notice_headline({"outcome": "BAD", "line": "refuted"}) == "✗ refuted" + + +def test_notice_headline_without_outcome_is_plain_line(): + assert _notice_headline({"line": "building…"}) == "building…" + + +def test_notice_headline_glyphs_come_from_the_reports_own_table(): + # One table, not a copy: a ✓ must mean the same thing in the callout, the console rollup and the + # HTML report. + assert all( + _notice_headline({"outcome": o.value, "line": "x"}) == f"{outcome_glyph(o)} x" + for o in Outcome + ) + + +def test_an_outcome_the_host_does_not_know_goes_unmarked(): + # A wheel emitting a label from a newer SDK loses its glyph, not its line. + assert _notice_headline({"outcome": "FLAKY", "line": "odd"}) == "odd" + + +class _RecordingHandler(GenericRustTaskHandler): + """Records where each event is routed, bypassing real textual mounting.""" + + def __init__(self, event_kinds: set[str], notice_kinds: set[str]): + super().__init__( + "t", "Label", cast(Any, None), cast(Any, None), ToolDisplayConfig(), + event_kinds, notice_kinds, + ) + self.notices: list[str] = [] + self.logged: list[str] = [] + + async def post_notice(self, headline, detail=None, *, toast=True): # type: ignore[override] + self.notices.append(headline if isinstance(headline, str) else headline.plain) + + async def _ensure_event_log(self): # type: ignore[override] + handler = self + + class _Log: + def write(self, line: str) -> None: + handler.logged.append(line) + + return _Log() + + +def _handle(handler: _RecordingHandler, payload: dict) -> None: + asyncio.run(handler.handle_event(payload, ["t"], "cp")) + + +def test_notice_kind_routes_to_post_notice_not_log(): + h = _RecordingHandler(event_kinds={"fuzz_pulse", "verdict"}, notice_kinds={"verdict"}) + _handle(h, {"type": "verdict", "outcome": "BAD", "line": "counterexample found"}) + assert h.notices == ["✗ counterexample found"] + assert h.logged == [] + + +def test_streaming_kind_routes_to_log_not_notice(): + h = _RecordingHandler(event_kinds={"fuzz_pulse", "verdict"}, notice_kinds={"verdict"}) + _handle(h, {"type": "fuzz_pulse", "line": "fuzzing…"}) + assert h.logged == ["[fuzz_pulse] fuzzing…"] + assert h.notices == [] + + +def test_undeclared_kind_is_ignored(): + h = _RecordingHandler(event_kinds={"verdict"}, notice_kinds={"verdict"}) + _handle(h, {"type": "mystery", "line": "nope"}) + assert h.notices == [] and h.logged == [] diff --git a/tests/test_rust_llm_agent.py b/tests/test_rust_llm_agent.py new file mode 100644 index 00000000..84158cd1 --- /dev/null +++ b/tests/test_rust_llm_agent.py @@ -0,0 +1,196 @@ +"""The Rust authoring session's prompt handling (no wheel / LLM needed). + +Two halves make up an author's system prompt, and the split is the point: the *host* owns the +protocol half — the tools, what the publish gate requires, what a skip and a give-up mean — and the +wheel owns the domain half. A wheel that hand-rolled the protocol could drift from what the host +actually enforces, so it is never asked to. + +The wheel's payload is a :class:`composer.rustapp.wire.Prompt`, parsed at the seam, so the shape is +checked where it crosses rather than read key-by-key later: a wheel that sends no ``instruction`` +fails here with the field named, rather than prompting the agent with a JSON dump of whatever it +did send. +""" + +import json +import pathlib +from typing import Any, cast + +import pytest +from pydantic import ValidationError + +from composer.rustapp.descriptor import AppDescriptor +from composer.rustapp.session import ( + CheckVocab, GateDeps, ProtocolTemplate, PublishDeps, _expect_tools, _feedback_tool, + _initial_prompt, _map_tool, _publish_tool, _validate_tool, rebuttal_model, +) +from composer.rustapp.wire import Prompt, parse_prompt +from composer.templates.loader import load_jinja_template +from tests.conftest import wire_descriptor, wire_prompt + + +def _descriptor(**overrides) -> AppDescriptor: + return AppDescriptor.model_validate(wire_descriptor(**overrides)) + + +def _protocol(*, gate_tool="validate_spec", has_judge=True, has_checks=True) -> str: + return ProtocolTemplate.bind({ + "gate_tool": gate_tool, + "has_judge": has_judge, + "has_checks": has_checks, + "check_noun": "check", + }).render_to(load_jinja_template) + + +def test_instruction_is_taken_as_sent(): + assert parse_prompt(json.dumps(wire_prompt("author X"))).instruction == "author X" + + +def test_no_system_prompt_declared_means_the_protocol_stands_alone(): + # ``None`` is the wheel saying it has nothing domain-specific to add. + assert parse_prompt(json.dumps(wire_prompt("author X"))).system is None + + +def test_backend_may_define_its_own_system_prompt(): + prompt = parse_prompt(json.dumps(wire_prompt("author X", "you are a fuzz author"))) + assert prompt == Prompt(system="you are a fuzz author", instruction="author X") + + +def test_a_payload_with_no_instruction_is_rejected_at_the_seam(): + with pytest.raises(ValidationError, match="instruction"): + parse_prompt('{"system": "you are a fuzz author"}') + + +def test_the_protocol_half_is_backend_agnostic(): + # It describes the session, not a language or a checker: every wheel gets the same text. + text = _protocol() + assert "Rust" not in text and "cargo" not in text + for tool in ("put_spec", "edit_spec", "get_spec", "result", "give_up"): + assert tool in text + + +def test_the_protocol_names_the_gate_the_session_actually_bound(): + # The gate differs per session kind; naming the wrong one would send the author looking for a + # tool that isn't on its belt. + assert "validate_spec" in _protocol(gate_tool="validate_spec") + assert "compile_spec" in _protocol(gate_tool="compile_spec", has_checks=False) + + +def test_a_session_with_no_judge_is_not_told_to_seek_review(): + assert "feedback_tool" not in _protocol(has_judge=False) + assert "feedback_tool" in _protocol(has_judge=True) + + +def test_a_setup_session_is_told_nothing_about_skips_or_expected_failures(): + # It formalizes no properties of its own, so it has nothing to skip and no unit to mark. + setup = _protocol(gate_tool="compile_spec", has_checks=False) + assert "record_skip" not in setup and "expect_check_failure" not in setup + + +_GENERIC = CheckVocab("check", "checks") + + +def test_the_initial_prompt_states_the_obligation_the_gate_will_enforce(): + # The names are the author's to choose, but coverage and honesty about them are enforced the + # same way for every backend — so the host words that once, rather than trusting each wheel's + # prose to say it (or to say it compatibly). + prompt = Prompt(system=None, instruction="Author the harness.") + text = _initial_prompt(prompt, True, _GENERIC) + assert "Author the harness." in text + assert "map_checks" in text + assert "skipped" in text and "really be in your spec" in text + + +def test_a_setup_sessions_initial_prompt_is_the_wheels_own(): + # Nothing to declare: a setup spec formalizes no properties, so there is no mapping at all. + prompt = Prompt(system=None, instruction="Author the fixture.") + assert _initial_prompt(prompt, False, _GENERIC) == "Author the fixture." + + +def test_the_obligation_speaks_the_wheels_own_noun(): + # A Crucible author reads about harness functions, not about "checks" — the word is the wheel's + # (AppDescriptor.check_noun), and it is what its own prompts and generated code already use. + text = _initial_prompt( + Prompt(system=None, instruction="Author it."), + True, + CheckVocab("harness function", "harness functions"), + ) + assert "harness functions verify which property" in text + # The tool NAME is fixed (`map_checks`, like `expect_check_failure`); it is the prose that has + # to speak the wheel's word, so the generic noun must not appear as a word of its own. + assert "check " not in text and "checks " not in text + + +# --------------------------------------------------------------------------- +# The tool surface speaks the wheel's vocabulary too +# --------------------------------------------------------------------------- + +def _crucible() -> CheckVocab: + return CheckVocab("harness function", "harness functions") + + +def _tool_text(tool) -> str: + """Everything about a tool the model actually reads: its description and every argument's.""" + schema = tool.tool_call_schema.model_json_schema() + parts = [tool.description, *( + p.get("description", "") for p in schema.get("properties", {}).values() + )] + parts += [ + p.get("description", "") + for d in schema.get("$defs", {}).values() + for p in d.get("properties", {}).values() + ] + return "\n".join(parts) + + +def _gate_deps(vocab: CheckVocab) -> GateDeps: + return GateDeps( + module=cast(Any, None), input_json="{}", workdir=pathlib.Path("."), + sandbox_json="{}", emit=lambda _k, _p: None, vocab=vocab, + ) + + +def test_the_gate_tool_describes_itself_in_the_wheels_noun(): + vocab = _crucible() + text = _tool_text(_validate_tool(_gate_deps(vocab), vocab)) + assert "harness function" in text + # `checks` survives as the *argument* name — the tools keep their generic API so the protocol + # can name them literally; it is the prose that speaks the wheel's language. + assert "check " not in text and "checks " not in text + + +def test_the_expected_failure_tools_and_the_publish_mapping_follow(): + vocab = _crucible() + fail, passage = _expect_tools(vocab) + publish = _publish_tool(PublishDeps(titles=[])) + for tool in (fail, passage, _map_tool(vocab)): + assert "harness function" in _tool_text(tool) + # The publish tool no longer carries the mapping — it is declared before the run, not after — + # so what it must still say in the wheel's words is what the gate refuses over. + assert "harness function" in _tool_text(publish) or "mapping" in _tool_text(publish) + + +def test_a_wheel_that_declares_no_noun_gets_the_generic_one(): + # `check_noun` is optional; a wheel with no better word gets the framework's. + assert CheckVocab.of(_descriptor(check_noun=None)).one == "check" + assert CheckVocab.of(_descriptor(check_noun="invariant")).many == "invariants" + + +def test_a_templated_tool_is_not_silently_the_base_one(): + # A family instantiated with different nouns must produce different LLM-facing text. This is + # the guard that ``with_template`` actually rewrote the schema, rather than handing back the + # untemplated ``{check}`` placeholders. + generic = _tool_text(_validate_tool(_gate_deps(_GENERIC), _GENERIC)) + crucible = _tool_text(_validate_tool(_gate_deps(_crucible()), _crucible())) + assert generic != crucible + assert "{check}" not in generic and "{check}" not in crucible + + +def test_the_rebuttal_tool_carries_the_wheels_declared_evidence_kinds(): + # Same failure mode, on the tool that was already built by subclassing: if the base schema wins, + # the judge is offered an untyped rebuttal instead of the wheel's closed set. + async def _judge(*_a): + raise AssertionError("not invoked") + + tool = _feedback_tool(_judge, rebuttal_model(["build_failure", "reasoned"])) + evidence = tool.tool_call_schema.model_json_schema()["$defs"]["Rebuttal"]["properties"] + assert evidence["evidence_type"]["enum"] == ["build_failure", "reasoned"] diff --git a/tests/test_rustapp.py b/tests/test_rustapp.py new file mode 100644 index 00000000..8df4c844 --- /dev/null +++ b/tests/test_rustapp.py @@ -0,0 +1,335 @@ +"""End-to-end tests for the Rust application/backend framework (composer.rustapp). + +These drive the ``echoprover`` demo wheel (built from ``rust/example-app``) as a +:class:`~autoprover_sdk.Backend`: the pure callouts (``descriptor`` / ``target_for`` / +``author_prompt`` / ``compile`` / ``validate``) plus the descriptor synthesis and the +host wiring. They need the ``echoprover`` wheel importable — ``uv sync`` builds it (the +``apps`` group, pulled in via ``dev``); tests skip cleanly otherwise. + +No Postgres / LLM is required — the callouts are pure (echoprover's ``compile`` is a +no-op and reading the spec is its whole checker), which is the point of the +passive-service design: the loop lives in Python and the wheel just answers questions. +""" + +import json + +import pytest +from pydantic import ValidationError + +echoprover = pytest.importorskip( + "echoprover", + reason="demo wheel not built; run `uv sync` (builds rust/example-app)", +) + +from composer.rustapp.descriptor import AppDescriptor, PhaseRole +from composer.rustapp.result import RustFormalResult +from composer.rustapp.wire import ComponentInput, Property, Target, Check, Verdict +from composer.authoring.state import SkippedProperty +from composer.spec.source.report.schema import Outcome +from tests.conftest import wire_verdict + + +def _component_input(*titles: str) -> str: + return ComponentInput( + program="Counter", + unit={"name": "Counter"}, + props=[ + Property( + component="Counter", title=t, sort="invariant", description="x", + slug=t.replace(" ", "_"), + ) + for t in titles + ], + ).model_dump_json() + + +def _target(*checks: str) -> str: + """A target and the report rows it covers — what the host passes ``validate``.""" + return Target( + name=checks[0], checks=[Check(name=c, properties=["p"], target=None) for c in checks] + ).model_dump_json() + + +def _sandbox() -> str: + """A passthrough ``Sandbox``: no confinement wrapper (``composer.sandbox.config.BackendSpec``).""" + return json.dumps({"argv_prefix": [], "timeout_s": 600}) + + +def test_descriptor_parses_and_maps_core_phases(): + desc = AppDescriptor.model_validate_json(echoprover.descriptor()) + assert desc.name == "echoprover" + # A tag from the closed ``ReportBackend`` set: the demo borrows the prover's outcome wording + # rather than inventing a vocabulary the report can't render. Anything outside the set fails + # here, at descriptor load, rather than later when the formalizer is constructed. + assert desc.backend_tag == "prover" + # Every *required* slot is mapped, plus a UI-only "solving" phase. The optional DISCOVERY slot is + # left unclaimed, which is the common case: the design-doc task then groups under the first phase. + slots = desc.role_map() + assert set(slots) == set(PhaseRole.required()) + assert PhaseRole.DISCOVERY not in slots + keys = [p.key for p in desc.ordered_phases()] + assert keys == ["analysis", "extraction", "solving", "formalization", "report"] + + +def test_each_declared_check_is_its_own_target_by_default(): + # The demo groups nothing, so every check the author declares is its own invocation. `None`, + # not a name — the host reads that as "its own target" without the wheel spelling one. + assert echoprover.target_for(_component_input("increment_increases"), "rule_x") is None + + +def test_author_prompt_lists_the_properties(): + prompt = json.loads(echoprover.author_prompt(_component_input("increment_increases"))) + assert "increment_increases" in prompt["instruction"] + assert prompt.get("system") is None + + +def test_compile_is_a_noop_ok(): + # The demo accepts any well-formed spec — compile is a no-op gate. + r = json.loads(echoprover.compile(_component_input("p"), "spec", "/tmp", _sandbox())) + assert r == {"status": "ok"} + + +def test_compile_takes_no_spec_at_all_for_a_preflight(): + # The preflight has nothing authored to build, so the callout crosses the FFI with `None` — + # what tells a wheel whose toolchain writes the spec to a file to render its own skeleton + # instead of writing an empty one. + r = json.loads(echoprover.compile(_component_input("p"), None, "/tmp", _sandbox())) + assert r == {"status": "ok"} + + +def test_validate_returns_a_verdict_for_every_row_the_target_covers(): + sandbox = _sandbox() + spec = "rule rule_p: ok\nrule rule_a: ok\nrule rule_b: ok" + res = json.loads( + echoprover.validate(_component_input("p"), spec, _target("rule_p"), "/tmp", sandbox) + ) + # ValidateOutcome: the spec declares the rule, so per-unit verdicts (not build_failed). + assert res == {"kind": "verdicts", "verdicts": [["rule_p", wire_verdict("GOOD")]]} + + # A target covering several rows answers for all of them in the one run — the wheel keys the + # verdicts off the units the host sent, so it never has to spell a unit name itself. + shared = json.loads( + echoprover.validate(_component_input("p"), spec, _target("rule_a", "rule_b"), "/tmp", sandbox) + ) + assert [u for u, _v in shared["verdicts"]] == ["rule_a", "rule_b"] + + +def test_a_declared_check_the_spec_does_not_contain_does_not_pass(): + # The names are the author's, so validate is what holds them to the artifact: a rule nobody + # wrote has nothing behind it and must not stamp a property as verified. + res = json.loads( + echoprover.validate(_component_input("p"), "rule rule_p: ok", _target("rule_ghost"), + "/tmp", _sandbox()) + ) + (name, verdict), = res["verdicts"] + assert name == "rule_ghost" + assert verdict["outcome"] == "ERROR" and "rule_ghost" in (verdict["detail"] or "") + + +def test_result_round_trips_through_cache_serialization(): + # The driver caches by model_dump/validate, so everything the loop accumulates has to survive + # that — including the nested per-check verdicts the wheel published. + res = RustFormalResult( + commentary="c", + artifact_text="spec", + checks=[("p", ["rule_p"])], + skipped=[SkippedProperty(property_title="q", reason="n/a")], + output_link="local://x", + verdicts={"rule_p": Verdict(outcome=Outcome.BAD, line=7, detail="counterexample", + duration_seconds=None, unit_file=None)}, + ) + reloaded = RustFormalResult.model_validate_json(res.model_dump_json()) + assert reloaded.property_checks() == [("p", ["rule_p"])] + assert reloaded.artifact_text == "spec" + assert reloaded.skipped[0].property_title == "q" + assert reloaded.verdicts["rule_p"].outcome is Outcome.BAD + assert reloaded.verdicts["rule_p"].detail == "counterexample" + + +# --------------------------------------------------------------------------- +# Generic host: entry point (argparse), shared-enum identity, frontend. +# These import the heavier host (needs the full composer stack). If it can't +# import (e.g. running against a slim env), skip rather than error. +# --------------------------------------------------------------------------- + +host = pytest.importorskip( + "composer.rustapp.host", reason="needs the full composer stack installed" +) + + +def test_entry_argparser_has_positionals_and_declared_flags(): + from composer.rustapp.entry import build_arg_parser + + app = host.build_application("echoprover") + parser = build_arg_parser(app) + + # Declared flag default (from the descriptor's ArgSpec) is applied. + args = parser.parse_args(["/proj", "src/C.sol:C", "doc.md"]) + assert args.project_root == "/proj" + assert args.main_contract == "src/C.sol:C" + assert args.system_doc == "doc.md" + assert args.max_concurrent == 4 + assert args.echo_tag == "demo" + + # …and is overridable. + args2 = parser.parse_args(["/proj", "src/C.sol:C", "doc.md", "--echo-tag", "hi"]) + assert args2.echo_tag == "hi" + + +def test_the_parser_the_entry_point_runs_is_the_one_that_carries_help_text(): + # ``build_arg_parser`` is the single definition ``rust_entry_point`` runs, not a hand-copy of an + # inline one — a second copy drifts, and a copy missing its help strings makes ``--help`` from + # the introspection path document nothing. + from composer.rustapp.entry import build_arg_parser + + # argparse re-wraps help to the terminal width, so compare on collapsed whitespace. + help_text = " ".join(build_arg_parser(host.build_application("echoprover")).format_help().split()) + # Hyphenated words are what argparse breaks *across* lines, so these fragments avoid them. + for expected in ( + "Project root", + "Main contract as path:ContractName", + "Path to the design document", + "Max concurrent agents", + "Cache namespace", + "Memory namespace", + "Interactively refine extracted properties", + "rounds per component", + "Max graph iterations", + ): + assert expected in help_text, expected + + +def test_declared_flags_are_threaded_by_dest_and_nothing_else_is(): + # What reaches ``validate_preconditions`` / every component's context: the descriptor's own + # flags, keyed by the dest argparse gave them — not the host's built-in options. + from composer.rustapp.entry import _declared_args, build_arg_parser + + app = host.build_application("echoprover") + ns = build_arg_parser(app).parse_args(["/proj", "src/C.sol:C", "doc.md", "--echo-tag", "hi"]) + assert _declared_args(ns, app.descriptor.args) == {"echo_tag": "hi"} + + +def test_the_unit_noun_defaults_and_pluralizes(): + desc = AppDescriptor.model_validate_json(echoprover.descriptor()) + assert desc.component_noun is None # the demo declares none + assert desc.unit_noun() == "component" + assert desc.unit_noun(plural=True) == "components" + named = desc.model_copy(update={"component_noun": "instruction"}) + assert named.unit_noun() == "instruction" + assert named.unit_noun(plural=True) == "instructions" + + +def test_system_doc_is_optional_with_discovery_phase_fallback(): + from composer.rustapp.entry import _discovery_phase, build_arg_parser + + app = host.build_application("echoprover") + parser = build_arg_parser(app) + + # system_doc may be omitted (→ discovery); still parses. + ns = parser.parse_args(["/proj", "src/C.sol:C"]) + assert ns.system_doc is None + assert parser.parse_args(["/proj", "src/C.sol:C", "doc.md"]).system_doc == "doc.md" + + # A wheel that declares no discover_design_doc phase falls back to its first phase. + first_key = app.descriptor.ordered_phases()[0].key + assert _discovery_phase(app) is app.phases.member(first_key) + + +def test_frontend_labels_and_backend_phases_share_one_enum(): + # The correctness invariant: the phases the driver stamps on TaskInfo (from + # the backend's core_phases) must be the SAME enum members the frontend's + # phase_labels are keyed by, or label lookup silently misses. + from composer.input.files import InMemoryTextFile + from composer.llm.anthropic import AnthropicRenderer + from composer.spec.context import SourceCode + from composer.spec.system_model import SolidityIdentifier + + app = host.build_application("echoprover") + source = SourceCode( + content=InMemoryTextFile( + basename="doc.md", string_contents="doc", renderer=AnthropicRenderer() + ), + project_root="/tmp/echo-proj", + contract_name=SolidityIdentifier("C"), + relative_path="src/C.sol", + forbidden_read="", + ) + backend = app.make_backend(source) + for slot, member in backend.core_phases.items(): + assert member in app.phases.labels, (slot, member) + # Section order lists every declared phase's label. + assert set(app.phases.section_order) == set(app.phases.labels.values()) + + +def test_generic_console_handler_renders_declared_events(capsys): + import asyncio + + from composer.rustapp.frontend import GenericRustConsoleHandler, _render_event + + assert _render_event({"type": "solver_line", "line": "hello"}) == "hello" + assert _render_event({"type": "x", "a": 1}) == '{"a": 1}' + + handler = GenericRustConsoleHandler({"solver_line"}) + asyncio.run(handler.handle_event({"type": "solver_line", "line": "L1"}, ["t"], "cp")) + # An undeclared kind is ignored. + asyncio.run(handler.handle_event({"type": "other", "line": "nope"}, ["t"], "cp")) + out = capsys.readouterr().out + assert "solver_line: L1" in out + assert "nope" not in out + + +def test_generic_tui_app_constructs(): + from composer.rustapp.frontend import GenericRustApp + + app = host.build_application("echoprover") + tui = GenericRustApp( + phase_labels=app.phases.labels, + section_order=app.phases.section_order, + header_text=app.header_text, + event_kinds={e.kind for e in app.descriptor.event_kinds}, + ) + assert tui is not None + + +def test_descriptor_carries_ecosystem_and_resolves(): + from composer.pipeline.ecosystem import EVM + from composer.rustapp.host import resolve_ecosystem + + desc = AppDescriptor.model_validate_json(echoprover.descriptor()) + assert desc.ecosystem == "evm" + assert resolve_ecosystem(desc) is EVM + + +def test_build_application_carries_resolved_ecosystem(): + from composer.pipeline.ecosystem import EVM + + app = host.build_application("echoprover") + assert app.ecosystem is EVM + + +def test_resolve_ecosystem_resolves_soroban(): + from composer.pipeline.ecosystem import SOROBAN + from composer.rustapp.host import resolve_ecosystem + + desc = AppDescriptor.model_validate_json(echoprover.descriptor()) + soroban = desc.model_copy(update={"ecosystem": "soroban"}) + assert resolve_ecosystem(soroban) is SOROBAN + + +def test_resolve_ecosystem_rejects_unregistered_chain(monkeypatch): + from composer.rustapp.host import resolve_ecosystem + + desc = AppDescriptor.model_validate_json(echoprover.descriptor()) + monkeypatch.setattr("composer.rustapp.host.ECOSYSTEMS", {}) + with pytest.raises(ValueError, match="not registered"): + resolve_ecosystem(desc) + + +def test_a_descriptor_missing_a_field_is_refused(): + # No wheel is old enough to be excused one — the SDK and the host ship together, so an absent + # `ecosystem` is a drifted mirror, and guessing "evm" would run the whole front half against the + # wrong system model and prompts. + raw = json.loads(echoprover.descriptor()) + del raw["ecosystem"] + with pytest.raises(ValidationError): + AppDescriptor.model_validate(raw) diff --git a/tests/test_rustapp_discovery_phase.py b/tests/test_rustapp_discovery_phase.py new file mode 100644 index 00000000..3b667edd --- /dev/null +++ b/tests/test_rustapp_discovery_phase.py @@ -0,0 +1,65 @@ +"""Which phase the design-doc discovery task is grouped under (``composer.rustapp.entry``). + +Discovery runs in the *entry point*, before the pipeline, and only when the doc wasn't passed on the +command line — so it is not one of the four phases the driver tags. A wheel that wants it in a +section of its own claims ``PhaseRole.DISCOVERY``, rather than spelling a magic phase key the host +recognizes by name and silently ignores when it is misspelled. + +No wheel and no services — the descriptor and the synthesized enum are all this needs. +""" + +import json +from typing import Any, cast + +from composer.rustapp.descriptor import AppDescriptor, PhaseRole +from composer.rustapp.entry import _discovery_phase +from composer.rustapp.host import ( + RustApplication, + build_phase_model, + resolve_ecosystem, +) +from tests.conftest import wire_descriptor, wire_phase + +PHASES = [ + {"key": "analysis", "label": "A", "order": 0, "role": "analysis"}, + {"key": "extraction", "label": "E", "order": 1, "role": "extraction"}, + {"key": "formalization", "label": "F", "order": 2, "role": "formalization"}, + {"key": "report", "label": "R", "order": 3, "role": "report"}, +] + + +def _app(*, claims_discovery: bool) -> RustApplication: + descriptor = AppDescriptor.model_validate( + wire_descriptor( + name="app", ecosystem="solana", + phases=[ + *PHASES, + wire_phase("find_doc", "Design Doc", 4, + "discovery" if claims_discovery else "grouping"), + ], + ) + ) + return RustApplication( + descriptor=descriptor, module=cast(Any, object()), + ecosystem=resolve_ecosystem(descriptor), phases=build_phase_model(descriptor), + ) + + +def test_the_discovery_task_uses_the_phase_that_claims_the_slot(): + app = _app(claims_discovery=True) + assert app.descriptor.role_map()[PhaseRole.DISCOVERY] == "find_doc" + assert _discovery_phase(app) is app.phases.member("find_doc") + + +def test_an_unclaimed_slot_falls_back_to_the_first_phase(): + # The common case: most wheels don't care where the task is grouped, and none has to know a + # magic key to opt in. + app = _app(claims_discovery=False) + assert PhaseRole.DISCOVERY not in app.descriptor.role_map() + assert _discovery_phase(app) is app.phases.member("analysis") + + +def test_the_optional_slot_is_not_required_of_every_application(): + # `build_phase_model` must keep demanding the four the driver tags — and only those. + app = _app(claims_discovery=False) + assert set(app.descriptor.role_map()) == set(PhaseRole.required()) diff --git a/tests/test_rustapp_gate.py b/tests/test_rustapp_gate.py new file mode 100644 index 00000000..fc498c2e --- /dev/null +++ b/tests/test_rustapp_gate.py @@ -0,0 +1,209 @@ +"""The Rust authoring session's publish gate (``composer.rustapp.session``). + +Publishing is gated on *stamps over the buffer as it now stands*, not on a Python retry loop. Three +things follow, and each is a rule this file pins down: + +* a checker run stamps the draft it saw, so any later edit silently invalidates it — the gate + refuses rather than publishing something nothing has checked; +* a check that failed blocks publishing unless the author marked it expected-to-fail with a reason, + which is how a real counterexample reaches the report as a finding rather than as noise; +* the declared property→checks mapping is checked against the checks the stamping run actually + covered, in both directions — the author names the checks, so the run is what holds those names + to the artifact. +""" + +from typing import Any, cast + +import pytest + +from composer.authoring.state import SkippedProperty, check_completion, spec_digest +from composer.rustapp.session import ( + FEEDBACK_KEY, VALIDATE_KEY, PropertyCheckMapping, RustSessionState, _unexplained, + _verdict_report, declared_checks, declared_names, targets_of, +) +from composer.rustapp.wire import Check, Outcome, Verdict + + +DRAFT = "fn c_no_free_mint(f: &mut Fixture) {}" + + +def _state(**kw) -> RustSessionState: + """The session state the gate reads (messages are irrelevant to it).""" + base = { + "curr_spec": DRAFT, + "skipped": [], + "validations": {}, + "required_validations": [VALIDATE_KEY], + "property_checks": [], + "expected_failures": {}, + "verdicts": {}, + "ran": [], + "failed": None, + } + return cast(RustSessionState, {**base, **kw}) + + +def _check(name: str, target: str | None = None, properties: list[str] | None = None) -> Check: + return Check(name=name, properties=properties or ["p"], target=target) + + +def _mapped(**by_title: list[str]) -> list[PropertyCheckMapping]: + return [PropertyCheckMapping(property_title=t, checks=c) for t, c in by_title.items()] + + +def _verdict(outcome: Outcome, detail: str | None = None) -> Verdict: + return Verdict(outcome=outcome, line=None, duration_seconds=None, unit_file=None, detail=detail) + + +# --------------------------------------------------------------------------- +# Stamps go stale +# --------------------------------------------------------------------------- + +def test_a_clean_run_satisfies_the_gate_for_the_draft_it_saw(): + stamped = _state(validations={VALIDATE_KEY: spec_digest(DRAFT, [])}) + assert check_completion(stamped) is None + + +def test_editing_the_spec_after_a_clean_run_invalidates_it(): + # The whole reason the stamp is a digest: nothing has to remember to clear it. + stamped = _state( + curr_spec=DRAFT + "\n// one more line", + validations={VALIDATE_KEY: spec_digest(DRAFT, [])}, + ) + assert "stale" in (check_completion(stamped) or "") + + +def test_declaring_a_skip_after_a_clean_run_also_invalidates_it(): + # A skip is part of what was reviewed — "this property is left out, here is why" is a claim the + # gate must not carry over from a draft that didn't make it. + stamped = _state( + skipped=[SkippedProperty(property_title="no_free_mint", reason="no oracle")], + validations={VALIDATE_KEY: spec_digest(DRAFT, [])}, + ) + assert "stale" in (check_completion(stamped) or "") + + +def test_every_required_check_must_have_stamped_the_current_draft(): + # A wheel that declares a judge requires both; a validate stamp alone is not enough. + both = _state( + required_validations=[VALIDATE_KEY, FEEDBACK_KEY], + validations={VALIDATE_KEY: spec_digest(DRAFT, [])}, + ) + assert FEEDBACK_KEY in (check_completion(both) or "") + + +def test_nothing_written_is_reported_as_such_rather_than_as_a_stale_stamp(): + assert "no spec written yet" in (check_completion(_state(curr_spec=None)) or "") + + +# --------------------------------------------------------------------------- +# A failing check blocks, unless it is the finding +# --------------------------------------------------------------------------- + +def test_a_failing_check_blocks_the_gate(): + verdicts = {"c_no_free_mint": _verdict(Outcome.BAD, "counterexample: mint(0)")} + assert _unexplained(verdicts, {}) == {"c_no_free_mint"} + + +def test_a_check_marked_expected_to_fail_does_not_block(): + # The counterexample IS the finding. Marking it is what turns a blocked run into a reported one, + # and the reason is what a human reads next to it. + verdicts = {"c_no_free_mint": _verdict(Outcome.BAD, "counterexample: mint(0)")} + assert _unexplained(verdicts, {"c_no_free_mint": "the program really does allow this"}) == set() + + +def test_every_non_good_outcome_blocks_not_just_a_refutation(): + # An ERROR or a TIMEOUT is not a passing check; treating only BAD as a failure would publish a + # check nothing actually decided. + for outcome in (Outcome.ERROR, Outcome.TIMEOUT, Outcome.UNKNOWN): + assert _unexplained({"c_x": _verdict(outcome)}, {}) == {"c_x"} + + +def test_the_report_says_which_checks_were_expected_to_fail(): + report = _verdict_report( + {"c_a": _verdict(Outcome.GOOD), "c_b": _verdict(Outcome.BAD, "cex")}, + {"c_b": "known bug"}, + ) + assert "c_a: GOOD" in report + assert "(expected to fail)" in report and "cex" in report + + +# --------------------------------------------------------------------------- +# What a session is still expected to check +# --------------------------------------------------------------------------- + +def test_the_declared_names_are_what_runs_deduplicated(): + # One check discharging three properties is one thing to run, not three. The mapping is + # many-to-many; the work list is its distinct names. + mapping = _mapped(a=["c_shared"], b=["c_shared", "c_b"], c=["c_shared"]) + assert declared_names(mapping) == ["c_shared", "c_b"] + + +def test_a_declared_check_is_paired_with_the_grouping_the_wheel_gives_it(): + # The two halves of a check come from the two parties that can know them: the name from the + # author, the invocation it runs under from the wheel. + class _Wheel: + def target_for(self, _input_json: str, check: str) -> str | None: + return "shared" if check != "c_own" else None + + checks = declared_checks(cast(Any, _Wheel()), "{}", _mapped(a=["c_a"], b=["c_own"])) + assert [(c.name, c.target) for c in checks] == [("c_a", "shared"), ("c_own", None)] + # …and each carries the author's claim about it, so a backend whose diagnostics speak in + # properties can place a finding without the host parsing anything. + assert [c.properties for c in checks] == [["a"], ["b"]] + + +def test_a_checks_target_defaults_to_its_own_name(): + assert [t.name for t in targets_of([_check("c_p")])] == ["c_p"] + + +def test_checks_sharing_a_target_are_run_once_and_carry_their_own(): + # A wheel may check a whole property set in one run; the host groups, so the wheel does not have + # to re-derive the grouping it was already given. + checks = [_check("c_a", "shared"), _check("c_b", "shared"), _check("c_c")] + targets = targets_of(checks) + assert [t.name for t in targets] == ["shared", "c_c"] + assert [c.name for c in targets[0].checks] == ["c_a", "c_b"] + + +def test_a_check_the_run_never_covered_cannot_be_claimed(): + # The author names the checks, so nothing but the run can say a name is real. A name that no + # target covered is one the wheel never answered for. + from composer.authoring.state import validate_check_mapping + from composer.rustapp.session import _MAPPING + + err = validate_check_mapping( + [(m.property_title, m.checks) for m in _mapped(no_free_mint=["c_invented"])], + [], ["no_free_mint"], _MAPPING, ran=["c_no_free_mint"], + ) + assert err is not None and "c_invented" in err + + +def test_one_check_may_be_claimed_by_several_properties(): + # A single rule discharging three related invariants: three report rows, one thing that ran. + from composer.authoring.state import validate_check_mapping + from composer.rustapp.session import _MAPPING + + err = validate_check_mapping( + [(m.property_title, m.checks) for m in _mapped(a=["c_all"], b=["c_all"], c=["c_all"])], + [], ["a", "b", "c"], _MAPPING, ran=["c_all"], + ) + assert err is None + + +def test_a_property_that_is_neither_skipped_nor_mapped_is_refused(): + from composer.authoring.state import validate_check_mapping + from composer.rustapp.session import _MAPPING + + err = validate_check_mapping( + [], [], ["no_free_mint"], _MAPPING, ran=["c_no_free_mint"], + ) + assert err is not None and "neither skipped nor mapped" in err + + +@pytest.mark.parametrize("outcome", [Outcome.GOOD, Outcome.BAD]) +def test_verdicts_are_recorded_verbatim(outcome: Outcome): + # Attribution is the wheel's — it owns its result format, so it decides which check a + # counterexample belongs to and the host records the answer without reinterpreting it. + v = _verdict(outcome, "as the wheel said it") + assert v.outcome is outcome and v.detail == "as the wheel said it" diff --git a/tests/test_rustapp_preflight.py b/tests/test_rustapp_preflight.py new file mode 100644 index 00000000..57eabdda --- /dev/null +++ b/tests/test_rustapp_preflight.py @@ -0,0 +1,263 @@ +"""Tests for the backend preflight: prepare the workspace, then *gate* it — before any property +exists, concurrently with system analysis (``composer.rustapp.adapter``). + +The gate is what makes the prep mean something. Placing a manifest and running ``cargo fetch`` +resolves a dependency graph but compiles nothing, and warming is deliberately best-effort — so +until this existed, the first thing that actually *built* the workspace was the compile of the first +LLM-authored draft, at the far end of the extraction phase. A dependency graph that won't resolve, a +harness that won't link, or codegen the generator rejects would surface there as compiler errors +an authoring agent cannot fix (it does not own the project's build files) and would consume every one +of its revise attempts. + +So: the wheel renders its own skeleton, the host builds it through the same ``compile`` callout the +authored artifacts use, and a failure is terminal. Fake wheel, fake project toolchain — no toolchain, +no LLM. +""" + +import json +from pathlib import Path +from typing import Any, cast + +import pytest + +from composer.rustapp.adapter import PreflightFailed, ProjectFacts +from composer.rustapp.descriptor import AppDescriptor +from tests.conftest import ( + wire_descriptor, wire_phase, wire_required_phases, wire_workspace_prep, +) +from composer.rustapp.host import build_backend, build_phase_model +from composer.rustapp.toolchain import PROJECT_TOOLCHAINS +from composer.spec.context import SourceCode +from composer.spec.system_model import SolidityIdentifier + +pytestmark = pytest.mark.asyncio + +CRATE_DIR = "programs/lend" +#: What the chain's registered toolchain reports for this project. Framework-side this is an opaque +#: object — that these keys spell a Cargo crate is the chain implementation's business. +SOURCE_UNIT = {"dir": CRATE_DIR, "package": "example-lending", "lib": "example_lending"} +IDL_DEST = "fuzz/vault/idls/example_lending.json" +#: A prep request in the chain's own shape, and the facts carrying it out establishes. +BUILD = {"build_program": "example_lending"} +BUILD_WITH_IDL = {**BUILD, "idl_dest": IDL_DEST} +ADDR = "LendvUkXRmuDKxGCCFJra9uxWMdMooPEmJk3qp7Tg1Z" + + +class FakeWheel: + """A wheel with a fixed ``workspace_prep`` plan and a scripted ``compile``.""" + + def __init__(self, request: dict, *, compile_errors: str | None = None): + self._plan = wire_workspace_prep( + toolchain_request={"warm_dirs": ["fuzz/vault"], **request} + ) + self._compile_errors = compile_errors + #: Every ``compile`` call, as ``(input, spec)`` — the assertion surface for these tests. + self.compiles: list[tuple[dict, str | None]] = [] + + def workspace_prep(self, _input_json: str) -> str: + return json.dumps(self._plan) + + def compile( + self, input_json: str, spec: str | None, _workdir: str, _sandbox_json: str + ) -> str: + self.compiles.append((json.loads(input_json), spec)) + if self._compile_errors is None: + return json.dumps({"status": "ok"}) + return json.dumps({"status": "failed", "errors": self._compile_errors}) + + +def _descriptor(*, with_preflight: bool = True) -> AppDescriptor: + # Without the role claimed the phase only groups, which is how an application says it has no + # gate — the phase itself stays, so the two descriptors differ in exactly one thing. + gate = wire_phase("preflight", "Build Preflight", 4, "preflight" if with_preflight else "grouping") + return AppDescriptor.model_validate( + wire_descriptor(ecosystem="solana", phases=[*wire_required_phases(), gate]) + ) + + +class _Run: + """Just enough ``PipelineRun``: both runners await the job inline, and record which was used.""" + + def __init__(self, source: SourceCode): + self.source = source + self.env = None + self.ctx = None + self.agent_tasks: list[str] = [] + self.cpu_tasks: list[str] = [] + #: Every ``TaskInfo`` the backend built, in order — the phase member matters as much as the + #: id (see the phase-tagging test). + self.tasks: list[Any] = [] + + async def runner(self, task_info, job): + self.agent_tasks.append(task_info.task_id) + self.tasks.append(task_info) + return await job() + + async def cpu_runner(self, task_info, job): + self.cpu_tasks.append(task_info.task_id) + self.tasks.append(task_info) + return await job() + + +def _source(root: Path) -> SourceCode: + return SourceCode( + content=None, # type: ignore[arg-type] — unused by preflight + project_root=str(root), + contract_name=SolidityIdentifier("vault"), + relative_path=f"{CRATE_DIR}/src/lib.rs", + forbidden_read="", + ) + + +def _project(root: Path) -> None: + """Just the source file the analysis identifier points at.""" + (root / CRATE_DIR / "src").mkdir(parents=True) + (root / CRATE_DIR / "src" / "lib.rs").write_text("// program") + + +class _FakeToolchain: + """A stand-in for the chain implementation the real Solana one is registered as + (``composer.rustapp.toolchain``). It does what any implementation must: read the request in its + own shape, do the work, and report what it established.""" + + def source_unit(self, _source): + return SOURCE_UNIT + + async def prepare(self, plan, _input, *, source, sandbox, timeout_s): + request = plan.toolchain_request + assert request["build_program"] # what these plans all ask for + if (idl_dest := request.get("idl_dest")) is None: + return {} + dest = Path(source.project_root) / idl_dest + dest.parent.mkdir(parents=True, exist_ok=True) + dest.write_text(json.dumps({"metadata": {"address": ADDR}})) + return {"idl": idl_dest} + + +def _fake_chain(monkeypatch): + monkeypatch.setitem(PROJECT_TOOLCHAINS, "solana", _FakeToolchain()) + + +async def _preflight(monkeypatch, tmp_path, wheel, *, with_preflight=True, declared=None): + _project(tmp_path) + _fake_chain(monkeypatch) + source = _source(tmp_path) + descriptor = _descriptor(with_preflight=with_preflight) + backend = build_backend(wheel, descriptor, source, phases=build_phase_model(descriptor)) + backend.declared_args = declared or {} + run = _Run(source) + return await backend.preflight(cast(Any, run)), run + + +async def test_the_gate_compiles_a_wheel_authored_skeleton_with_no_spec(tmp_path, monkeypatch): + wheel = FakeWheel(BUILD) + result, _run = await _preflight(monkeypatch, tmp_path, wheel) + + assert isinstance(result, ProjectFacts) + assert len(wheel.compiles) == 1 + gate_input, spec = wheel.compiles[0] + # `kind` is what the wheel dispatches on, and there is no spec — not an empty one, which a + # toolchain could take for a real spec file: nothing has been authored yet, so the wheel renders + # the skeleton itself. + assert gate_input["kind"] == "preflight" + assert spec is None + assert gate_input["props"] == [] + + +async def test_the_resolved_source_unit_is_carried_forward(tmp_path, monkeypatch): + wheel = FakeWheel(BUILD) + result, _run = await _preflight(monkeypatch, tmp_path, wheel) + + # Whatever the chain's toolchain reported — none of it follows from the analysis identifier + # ("vault"), and the gated build, every authoring turn and the deliverable must all agree on it. + assert result.source_unit == SOURCE_UNIT + assert wheel.compiles[0][0]["source_unit"] == SOURCE_UNIT + + +async def test_what_the_prep_established_reaches_the_gate_that_builds_against_it( + tmp_path, monkeypatch +): + wheel = FakeWheel(BUILD_WITH_IDL) + result, _run = await _preflight(monkeypatch, tmp_path, wheel) + + assert result.prep_facts == {"idl": IDL_DEST} + # The gate renders the same workspace the prep just set up, so it must see what the prep + # established — here, that the harness can generate its types from a placed file rather than + # linking the program's crate. + assert wheel.compiles[0][0]["prep_facts"] == {"idl": IDL_DEST} + + +async def test_a_prep_that_established_nothing_says_so_with_an_empty_answer(tmp_path, monkeypatch): + wheel = FakeWheel(BUILD) + result, _run = await _preflight(monkeypatch, tmp_path, wheel) + + # Empty is the whole spelling of "nothing established" — a fact present but empty would read as + # something being in place at an empty path. + assert result.prep_facts == {} + assert wheel.compiles[0][0]["prep_facts"] == {} + + +async def test_declared_args_are_in_scope_for_the_gate(tmp_path, monkeypatch): + # Prep may need one (Crucible reads `program_idl` when deciding how to source the types), so they + # are on the input from the very first callout. + wheel = FakeWheel(BUILD) + _result, _run = await _preflight( + monkeypatch, tmp_path, wheel, declared={"fuzz_timeout": 30} + ) + assert wheel.compiles[0][0]["args"]["fuzz_timeout"] == 30 + + +async def test_a_failing_gate_raises_with_the_diagnostics_and_does_not_retry(tmp_path, monkeypatch): + errors = "error[E0432]: unresolved import `example_lending::instruction`" + wheel = FakeWheel(BUILD, compile_errors=errors) + + with pytest.raises(PreflightFailed) as excinfo: + await _preflight(monkeypatch, tmp_path, wheel) + + assert errors in str(excinfo.value) + # One attempt. There is nothing to re-author: the failure is in the build files/toolchain, and the + # message says so rather than letting a later authoring loop discover it the expensive way. + assert len(wheel.compiles) == 1 + + +async def test_without_a_declared_preflight_the_prep_runs_but_nothing_is_gated(tmp_path, monkeypatch): + # The gate is opt-in per wheel; the workspace prep is not. + wheel = FakeWheel(BUILD_WITH_IDL) + result, run = await _preflight(monkeypatch, tmp_path, wheel, with_preflight=False) + + assert result.prep_facts == {"idl": IDL_DEST} # the prep still ran + assert wheel.compiles == [] + assert run.agent_tasks == [] and run.cpu_tasks == [] # the prep is silent, so there is no task + + +async def test_the_build_spends_a_cpu_slot_not_an_agent_slot(tmp_path, monkeypatch): + # The agent semaphore budgets concurrent *agents*; a multi-minute cargo build charged to it would + # silently take a quarter of the default concurrency away from the analysis it overlaps. It is + # throttled all the same — against the CPU budget, which is what it actually spends. + wheel = FakeWheel(BUILD) + _result, run = await _preflight(monkeypatch, tmp_path, wheel) + + assert run.cpu_tasks == ["demoprover-preflight"] + assert run.agent_tasks == [] + + +async def test_the_gate_is_tagged_with_the_declared_phase_member(tmp_path, monkeypatch): + # The task id comes from the step's kind, and the phase from its declared `phase_key` resolved + # against the backend's own synthesized enum. That identity is load-bearing: the frontend looks + # up section labels by enum *member*, so a member from any other copy of the enum would land the + # task in no section at all. `RustBackend.task_info` is the only thing that resolves it. + wheel = FakeWheel(BUILD) + _project(tmp_path) + _fake_chain(monkeypatch) + source = _source(tmp_path) + descriptor = _descriptor() + backend = build_backend(wheel, descriptor, source, phases=build_phase_model(descriptor)) + run = _Run(source) + await backend.preflight(cast(Any, run)) + + info = run.tasks[0] + assert info.task_id == "demoprover-preflight" + assert info.label == "Build Preflight" + assert info.phase is backend.phase["preflight"] + # …and it is the same enum the frontend's labels are keyed by, not a fresh synthesis of it. + assert type(info.phase) is backend.phase diff --git a/tests/test_rustapp_setup_cache.py b/tests/test_rustapp_setup_cache.py new file mode 100644 index 00000000..2d59af0e --- /dev/null +++ b/tests/test_rustapp_setup_cache.py @@ -0,0 +1,310 @@ +"""Tests for the shared setup spec: when it is authored, from what, and its cache. + +Three things are pinned here. **When**: not during ``prepare_formalization`` (which runs +concurrently with property extraction, so the properties don't exist yet) but in +``StagedFormalizer.begin`` — after extraction, before the per-unit fan-out, which is also the call +that produces the formalizer. **From what**: the union of *every* unit's +properties, not whichever unit happened to formalize first; the artifact is what makes those +properties checkable, so a multi-component run whose fixture only knew one component's properties +would tell the rest to work within a surface designed without them +(docs/crucible-component-units.md (PR3) §8.2). **Caching**: authoring it is a full LLM loop, on a large +program the longest single step of a run, so a re-run after something failed downstream must not pay +for it again. Like the driver's other caches it only stores when the run has a cache namespace +(``--cache-ns``); without one every step is recomputed, by design. + +Stubs throughout: the "author" step is a counter, the store is a dict. +""" + +import json +from dataclasses import dataclass +from typing import cast + +import pytest + +from composer.pipeline.ptypes import BackendJob +from composer.spec.types import PropertyFormulation + +import composer.rustapp.adapter as adapter +from composer.rustapp.adapter import ( + RustFormalizer, RustPreparedSystem, RustStagedFormalizer, _setup_identity +) +from composer.rustapp.descriptor import AppDescriptor +from composer.rustapp.session import SessionResult +from tests.conftest import wire_descriptor, wire_phase, wire_required_phases +from composer.rustapp.wire import Property, SetupInput +from composer.spec.context import WorkflowContext + +pytestmark = pytest.mark.asyncio + +FIXTURE = "// FIXTURE\nstruct Fixture {}" + + +class _Store: + """The subset of ``BaseStore`` the typed cache uses.""" + + def __init__(self): + self.items: dict[tuple, dict] = {} + + async def aget(self, ns, key): + value = self.items.get((tuple(ns), key)) + return None if value is None else type("Item", (), {"value": value})() + + async def aput(self, ns, key, value): + self.items[(tuple(ns), key)] = value + + async def adelete(self, ns, key): + self.items.pop((tuple(ns), key), None) + + +class _Run: + def __init__(self, ctx): + self.ctx = ctx + self.env = None + self.source = None + #: Every ``TaskInfo`` the backend built, in order. + self.tasks: list = [] + + async def runner(self, task_info, job): + self.tasks.append(task_info) + return await job() + + +def _descriptor() -> AppDescriptor: + return AppDescriptor.model_validate( + wire_descriptor( + ecosystem="solana", + phases=[ + *wire_required_phases(), + wire_phase("build_harness", "Build Harness", 4, "setup"), + ], + ) + ) + + +PROPS = [ + PropertyFormulation(title="no overflow", sort="invariant", description="balance never overflows") +] + + +@dataclass(frozen=True) +class _Unit: + """The two `FeatureUnit` members `begin` reads: the name of the unit a property was inferred for, + which is what each property carries onto the wire, and the unit's own semantic content, which is + what the run's unit set carries.""" + display_name: str + + def feature_json(self) -> dict[str, object]: + return {"slug": self.display_name} + + +def _jobs(*prop_lists: list[PropertyFormulation], names: list[str] | None = None) -> list[BackendJob]: + """One `BackendJob` per unit — what the driver hands `begin` after extraction.""" + units = names or [f"unit{i}" for i in range(len(prop_lists))] + return [ + BackendJob(feat=cast(object, _Unit(n)), props=p) for n, p in zip(units, prop_lists) + ] + + +async def _formalizer( + monkeypatch, ctx, authored: list[str], tmp_path, *, props=None, jobs=None, run=None +) -> RustFormalizer: + """Drive prepare→begin with the LLM authoring stubbed, returning the formalizer ``begin`` built + around the authored fixture.""" + from composer.rustapp.host import build_backend, build_phase_model + from composer.spec.context import SourceCode + from composer.spec.system_model import SolidityIdentifier + + async def fake_session(*, input, **_kw): + authored.append(input) + return SessionResult( + commentary="", spec=FIXTURE, skipped=[], property_checks=[], + verdicts={}, ran=[], expected_failures={}, + ) + + async def fake_prep(_module, _input, **_kw): + return {} # a plan that asked for nothing establishes nothing + + monkeypatch.setattr(adapter, "run_session", fake_session) + monkeypatch.setattr(adapter, "run_workspace_prep", fake_prep) + + source = SourceCode( + content=None, # type: ignore[arg-type] — unused by prepare_formalization + project_root=str(tmp_path), + contract_name=SolidityIdentifier("example_lending"), + relative_path="programs/lend/src/lib.rs", + forbidden_read="", + ) + descriptor = _descriptor() + backend = build_backend( + object(), # type: ignore[arg-type] — no callout + descriptor, source, phases=build_phase_model(descriptor), + ) + run = run or _Run(ctx) + run.source = source # type: ignore[assignment] + # The workspace prep happens in the backend's preflight (concurrently with analysis, before this + # point) and hands its outcome forward — here the stubbed "no IDL requested". + preflight = await backend.preflight(cast(object, run)) # type: ignore[arg-type] + prepared = RustPreparedSystem("main", backend, preflight) + before = len(authored) # callers reuse one list across repeat runs, so count this run's own + staged = await prepared.prepare_formalization(run) # type: ignore[arg-type] + # This run has authored nothing yet, and there is no formalizer to inspect — prep runs alongside + # extraction, so the properties aren't known here. A wheel that declares a `setup` step gets + # the staged type back, and `begin` is the only thing that can turn it into a formalizer. + assert isinstance(staged, RustStagedFormalizer) + assert len(authored) == before + return await staged.begin(jobs or _jobs(props if props is not None else PROPS), run) # type: ignore[arg-type] + + +async def _prepare(monkeypatch, ctx, authored: list[str], tmp_path, *, props=None, jobs=None) -> str | None: + """As :func:`_formalizer`, narrowed to the authored fixture itself.""" + f = await _formalizer(monkeypatch, ctx, authored, tmp_path, props=props, jobs=jobs) + return f._setup_result + + +def _ctx(store, *, namespace) -> WorkflowContext: + return WorkflowContext.create( + services=lambda _ns: None, # type: ignore[arg-type] + thread_id="t", store=store, recursion_limit=10, cache_namespace=namespace, + ) + + +async def test_the_setup_artifact_is_authored_once_and_then_reused(monkeypatch, tmp_path): + store, authored = _Store(), [] + first = await _prepare(monkeypatch, _ctx(store, namespace=("run-ns",)), authored, tmp_path) + assert first == FIXTURE and len(authored) == 1 + + # A second run with the same namespace (i.e. `--cache-ns` on both) reuses it: no second LLM loop. + second = await _prepare(monkeypatch, _ctx(store, namespace=("run-ns",)), authored, tmp_path) + assert second == FIXTURE and len(authored) == 1 + + +async def test_the_artifact_reaches_every_component_as_the_inputs_setup(monkeypatch, tmp_path): + # How the fixture actually gets to the components: `begin` builds the formalizer around the + # authored artifact, which it puts on every component's ``AuthorInput.setup`` — never assigned + # onto a formalizer that is already running. + store, authored = _Store(), [] + f = await _formalizer(monkeypatch, _ctx(store, namespace=None), authored, tmp_path) + assert f._setup_result == FIXTURE + + +async def test_the_setup_task_is_tagged_with_the_declared_phase_member(monkeypatch, tmp_path): + # The authoring loop runs as its own visible task, tagged with the phase the wheel declared for + # it ("build_harness" here) — resolved against the backend's own synthesized enum, since the + # frontend keys its section labels by enum member identity. `RustBackend.task_info` is what does + # that; the id half comes from the step's kind, not from a string at the call site. + store, authored = _Store(), [] + ctx = _ctx(store, namespace=None) + run = _Run(ctx) + f = await _formalizer(monkeypatch, ctx, authored, tmp_path, run=run) + assert f is not None + + info = run.tasks[0] + assert info.task_id == "demoprover-setup" + assert info.label == "Build Harness" + assert info.phase.name == "build_harness" + + +async def test_the_setup_artifact_is_authored_from_the_extracted_properties(monkeypatch, tmp_path): + # The point of deferring it: the prompt input carries the properties the fixture must support, + # so the fixture can expose an action per instruction they exercise instead of guessing. + store, authored = _Store(), [] + await _prepare(monkeypatch, _ctx(store, namespace=None), authored, tmp_path) + assert [p.title for p in authored[0].props] == ["no overflow"] + assert authored[0].kind == "setup" + + +async def test_the_artifact_is_authored_from_every_unit_s_properties(monkeypatch, tmp_path): + # The multi-component case, and the reason `begin` exists. With K units the fixture must be + # designed around ALL of their properties; authoring it from whichever unit formalized first + # would leave the other K-1 asserting against a surface built without them. + store, authored = _Store(), [] + deposits = [PropertyFormulation(title="deposit_conserves", sort="invariant", description="d")] + admin = [PropertyFormulation(title="only_admin_sets_fee", sort="safety_property", description="a")] + farms = [PropertyFormulation(title="stake_matches_position", sort="invariant", description="f")] + await _prepare( + monkeypatch, _ctx(store, namespace=None), authored, tmp_path, + jobs=_jobs(deposits, admin, farms), + ) + assert len(authored) == 1, "the shared artifact is authored once, not once per unit" + assert [p.title for p in authored[0].props] == [ + "deposit_conserves", "only_admin_sets_fee", "stake_matches_position" + ] + + +async def test_same_titled_properties_of_two_components_are_both_carried(monkeypatch, tmp_path): + # A title is unique only within the component it was inferred for (extraction validates that + # much), so two components can each carry a "solvency" property and mean different things by it. + # Both are properties the fixture has to make checkable, and both are properties their component + # will be asked to formalize — merging them by title would author the artifact around one. + store, authored = _Store(), [] + pool = PropertyFormulation(title="solvency", sort="invariant", description="the pool is solvent") + vault = PropertyFormulation(title="solvency", sort="invariant", description="the vault is solvent") + other = PropertyFormulation(title="only_admin", sort="safety_property", description="a") + await _prepare( + monkeypatch, _ctx(store, namespace=None), authored, tmp_path, + jobs=_jobs([pool], [vault, other], names=["Pool", "Vault"]), + ) + # Each carries the unit that inferred it, which is what tells the two "solvency" apart — the + # wheel is authoring one artifact for both and has to know whose surface each is stated over. + assert [(p.component, p.title) for p in authored[0].props] == [ + ("Pool", "solvency"), ("Vault", "solvency"), ("Vault", "only_admin"), + ] + # …and the wheel can still name a check per property: the host-assigned slugs stay distinct. + assert len({p.slug for p in authored[0].props}) == 3 + + +async def test_a_different_property_set_authors_a_different_artifact(monkeypatch, tmp_path): + # …and since the properties shape it, they are part of its cache identity. + store, authored = _Store(), [] + ns = ("run-ns",) + await _prepare(monkeypatch, _ctx(store, namespace=ns), authored, tmp_path) + other = [PropertyFormulation(title="other", sort="invariant", description="d")] + await _prepare(monkeypatch, _ctx(store, namespace=ns), authored, tmp_path, props=other) + assert len(authored) == 2 + + +async def test_without_a_cache_namespace_nothing_is_stored(monkeypatch, tmp_path): + # The default: `--cache-ns` absent → `cache_namespace=None` → every step is recomputed, and the + # store is never even written to. This is why a re-run repeats the whole pipeline. + store, authored = _Store(), [] + for _ in range(2): + assert await _prepare(monkeypatch, _ctx(store, namespace=None), authored, tmp_path) == FIXTURE + assert len(authored) == 2 + assert store.items == {} + + +@pytest.mark.filterwarnings("ignore::pytest.PytestWarning") +def test_the_setup_key_covers_what_it_is_authored_from_and_not_run_knobs(): + base = SetupInput( + program="example_lending", + source_unit={"dir": "programs/lend", "lib": "example_lending"}, + model={"programs": [{"name": "example_lending"}]}, + props=[ + Property(component="Lend", title="no overflow", sort="invariant", description="d") + ], + args={"fuzz_timeout": 30}, + prep_facts={"idl": "fuzz/x/idls/example_lending.json"}, + ) + same = _setup_identity(base) + + def varied(**update) -> str: + return _setup_identity(base.model_copy(update=update)) + + # A fuzz budget doesn't change what gets authored — keying on it would discard the artifact. + assert varied(args={"fuzz_timeout": 900}) == same + # Everything the prompt is built from does. + assert varied(program="other") != same + assert varied(model={"programs": []}) != same + assert varied(source_unit={"dir": "programs/other"}) != same + # …including the properties: they are what the fixture is designed around. + assert varied(props=[]) != same + # …down to the unit each was inferred for: the same title stated over another unit's surface is + # another property, and the fixture has to support it there too. + assert varied(props=[base.props[0].model_copy(update={"component": "Farms"})]) != same + # …including what the prep established, which is what decides where the types come from. + assert varied(prep_facts={}) != same + # Stable across key ordering *within* the opaque model — it arrives as JSON, and the wire model + # fixes the order of everything else. + model = {"programs": [{"name": "example_lending"}], "extra": {"a": 1, "b": 2}} + reordered = json.loads(json.dumps({k: model[k] for k in reversed(list(model))})) + assert varied(model=model) == varied(model=reordered) diff --git a/tests/test_rustapp_toolchain_sem.py b/tests/test_rustapp_toolchain_sem.py new file mode 100644 index 00000000..988240a9 --- /dev/null +++ b/tests/test_rustapp_toolchain_sem.py @@ -0,0 +1,59 @@ +"""The blocking-callout guard in ``composer.rustapp.session._blocking``. + +``compile``/``validate`` are synchronous wheel calls that spawn a toolchain (``run-confined``), so +they run in a worker thread. A wheel that declares ``serialize_toolchain`` shares one workdir / +cargo target across its units, and then those calls must not overlap — the semaphore is the only +thing keeping two concurrent components out of the same build directory. +""" + +import asyncio +import time + +import pytest + +from composer.rustapp.session import _blocking + + +@pytest.mark.asyncio +async def test_without_a_semaphore_the_call_just_runs_off_the_loop(): + assert await _blocking(lambda: "out", None) == "out" + + +@pytest.mark.asyncio +async def test_the_semaphore_is_released_so_later_calls_are_not_blocked(): + sem = asyncio.Semaphore(1) + assert await _blocking(lambda: "a", sem) == "a" + # Would hang here if the guard leaked the permit. + assert await _blocking(lambda: "b", sem) == "b" + assert not sem.locked() + + +@pytest.mark.asyncio +async def test_the_semaphore_keeps_concurrent_calls_out_of_the_shared_workdir(): + sem = asyncio.Semaphore(1) + live = 0 + peak = 0 + + def thunk() -> str: + nonlocal live, peak + live += 1 + peak = max(peak, live) + time.sleep(0.01) # long enough that unguarded threads would overlap + live -= 1 + return "done" + + results = await asyncio.gather(*(_blocking(thunk, sem) for _ in range(4))) + assert results == ["done"] * 4 + assert peak == 1 + + +@pytest.mark.asyncio +async def test_a_raising_callout_does_not_leave_the_semaphore_held(): + sem = asyncio.Semaphore(1) + + def boom() -> str: + raise RuntimeError("toolchain died") + + with pytest.raises(RuntimeError, match="toolchain died"): + await _blocking(boom, sem) + assert not sem.locked() diff --git a/tests/test_rustapp_validate_target.py b/tests/test_rustapp_validate_target.py new file mode 100644 index 00000000..c0ab046a --- /dev/null +++ b/tests/test_rustapp_validate_target.py @@ -0,0 +1,305 @@ +"""What the host hands ``validate``, and what it does with the answer. + +A report row is a *check*; the thing the host actually runs is a **target**, and several checks can +share one (Crucible checks a component's whole property set in a single fuzz run). The check names +are the author's — they come from the mapping ``map_checks`` declared — while the grouping is the +wheel's, one ``target_for`` answer per name. The host puts the two together and passes each target +with the checks it covers, rather than leaving the wheel to recover them by name. + +The grouping lives in the ``validate_spec`` tool the author calls, so these drive that tool directly +against a recording wheel. No LLM is involved. +""" + +import json +import pathlib +from dataclasses import dataclass, field +from typing import Any, cast + +import pytest +from langchain_core.messages import ToolMessage + +from composer.authoring.state import SkippedProperty, spec_digest +from composer.rustapp import adapter +from composer.rustapp.descriptor import AppDescriptor +from composer.rustapp.session import ( + VALIDATE_KEY, CheckVocab, GateDeps, PropertyCheckMapping, RustSessionState, SessionResult, + _validate_tool, +) +from composer.rustapp.wire import Target, Check, ValidateCoverageError +from composer.spec.source.report.schema import Outcome +from composer.spec.types import PropertyFormulation +from tests.conftest import wire_descriptor, wire_verdict + +SPEC = "fn c_farms(f: &mut Fixture) {}" + +#: What the author declared: two properties whose checks share a target, and a third that is its own. +MAPPING = [ + PropertyCheckMapping(property_title="stake matches", checks=["c_stake"]), + PropertyCheckMapping(property_title="no double stake", checks=["c_dbl"]), + PropertyCheckMapping(property_title="fees capped", checks=["c_fees"]), +] + +#: The wheel's grouping of those names — the other half of a ``Check``. +TARGETS = {"c_stake": "c_farms", "c_dbl": "c_farms", "c_fees": None} + + +@dataclass +class _Wheel: + """The callouts the gate reaches. ``validate`` answers GOOD for every covered unit.""" + + #: Every ``Target`` the host passed, in call order. + targets: list[Target] = field(default_factory=list) + outcome: str = "GOOD" + #: Checks the wheel leaves without a verdict though its target covers them. + omit: frozenset[str] = frozenset() + + def target_for(self, _input_json: str, check: str) -> str | None: + return TARGETS.get(check) + + def judge(self, _input_json: str) -> str | None: + return None # no judge, so no review machinery is bound + + def validate( + self, _input_json: str, _spec: str, target_json: str, _workdir: str, _sandbox: str + ) -> str: + target = Target.model_validate_json(target_json) + self.targets.append(target) + return json.dumps({ + "kind": "verdicts", + "verdicts": [ + [c.name, wire_verdict(self.outcome)] + for c in target.checks if c.name not in self.omit + ], + }) + + +def _state(**kw) -> RustSessionState: + base = { + "messages": [], + "curr_spec": SPEC, + "skipped": [], + "validations": {}, + "required_validations": [VALIDATE_KEY], + "property_checks": list(MAPPING), + "expected_failures": {}, + "verdicts": {}, + "ran": [], + "failed": None, + } + return cast(RustSessionState, {**base, **kw}) + + +def _deps(wheel: _Wheel, tmp_path: pathlib.Path) -> GateDeps: + return GateDeps( + module=cast(Any, wheel), + input_json="{}", + workdir=tmp_path, + sandbox_json="{}", + emit=lambda _kind, _payload: None, + ) + + +async def _validate(wheel: _Wheel, tmp_path: pathlib.Path, state=None, checks=None): + """Invoke the gate tool exactly as the graph would, and return what it handed back.""" + tool = _validate_tool(_deps(wheel, tmp_path), CheckVocab("check", "checks")) + out = await tool.ainvoke({ + "name": "validate_spec", + "type": "tool_call", + "id": "t1", + "args": { + "state": state if state is not None else _state(), + "tool_call_id": "t1", + "checks": checks, + }, + }) + # A tool that returns a plain string is wrapped in a ToolMessage by ``ainvoke``; one that + # returns a state update hands the Command back as-is. + return out.content if isinstance(out, ToolMessage) else out + + +@pytest.mark.asyncio +async def test_each_distinct_target_runs_once_carrying_the_checks_it_covers(tmp_path): + wheel = _Wheel() + await _validate(wheel, tmp_path) + + # One run per DISTINCT target, not per check — the two properties sharing `c_farms` are one run. + assert [t.name for t in wheel.targets] == ["c_farms", "c_fees"] + # …and each run is told exactly which checks it owes a verdict for, so the wheel never re-derives + # the grouping the host just computed. + assert [[c.name for c in t.checks] for t in wheel.targets] == [["c_stake", "c_dbl"], ["c_fees"]] + # …carrying the author's own claim about each, which is what a backend whose diagnostics speak + # in properties (Crucible's tagged assertions) places a counterexample by. + assert [c.properties for c in wheel.targets[0].checks] == [["stake matches"], ["no double stake"]] + + +@pytest.mark.asyncio +async def test_every_covered_check_gets_its_verdict_recorded(tmp_path): + # A shared target answers for several checks in one call; all of them must reach the state, or + # the report silently loses the checks that shared a run. + command = await _validate(_Wheel(), tmp_path) + + verdicts = command.update["verdicts"] + assert set(verdicts) == {"c_stake", "c_dbl", "c_fees"} + assert all(v.outcome is Outcome.GOOD for v in verdicts.values()) + + +@pytest.mark.asyncio +async def test_a_clean_full_run_stamps_the_draft_it_saw(tmp_path): + command = await _validate(_Wheel(), tmp_path) + assert command.update["validations"] == {VALIDATE_KEY: spec_digest(SPEC, [])} + + +@pytest.mark.asyncio +async def test_a_partial_run_never_stamps(tmp_path): + # Running a subset is for iterating on one problem. Letting it stamp would publish a spec whose + # other checks nothing had run. + out = await _validate(_Wheel(), tmp_path, checks=["c_fees"]) + assert isinstance(out, str) and "partial" in out + + +@pytest.mark.asyncio +async def test_a_partial_run_only_runs_the_targets_it_was_asked_for(tmp_path): + wheel = _Wheel() + await _validate(wheel, tmp_path, checks=["c_fees"]) + assert [t.name for t in wheel.targets] == ["c_fees"] + + +@pytest.mark.asyncio +async def test_an_unknown_check_name_is_refused_rather_than_silently_dropped(tmp_path): + wheel = _Wheel() + out = await _validate(wheel, tmp_path, checks=["c_invented"]) + assert isinstance(out, str) and "c_invented" in out + assert wheel.targets == [], "nothing ran" + + +@pytest.mark.asyncio +async def test_a_failing_check_leaves_the_gate_unstamped(tmp_path): + out = await _validate(_Wheel(outcome="BAD"), tmp_path) + assert isinstance(out, str) + assert "c_stake" in out and "NOT satisfied" in out + + +@pytest.mark.asyncio +async def test_a_failure_the_author_marked_as_the_finding_stamps(tmp_path): + # The counterexample IS the result. Marking it is what lets the run be published with the + # failure recorded and explained. + marked = _state(expected_failures={ + name: "the program really does allow it" for name in TARGETS + }) + command = await _validate(_Wheel(outcome="BAD"), tmp_path, state=marked) + assert VALIDATE_KEY in command.update["validations"] + + +@pytest.mark.asyncio +async def test_a_wheel_that_leaves_a_covered_check_unanswered_is_refused(tmp_path): + # An unanswered check is not a failing check: nothing downstream has anything to object to, so + # silence would read as a clean run. A wheel that answers for none of a target's checks is the + # extreme of it — the gate would stamp a component nothing had checked. + with pytest.raises(ValidateCoverageError, match="c_dbl"): + await _validate(_Wheel(omit=frozenset({"c_dbl"})), tmp_path) + with pytest.raises(ValidateCoverageError): + await _validate(_Wheel(omit=frozenset({"c_stake", "c_dbl", "c_fees"})), tmp_path) + + +@pytest.mark.asyncio +async def test_nothing_declared_means_nothing_runs(tmp_path): + # The declaration IS the work list. Running "everything" when the author has declared nothing + # would be running nothing while reporting a clean sweep, so the tool says what to do instead. + wheel = _Wheel() + out = await _validate(wheel, tmp_path, state=_state(property_checks=[])) + assert isinstance(out, str) and "map_checks" in out + assert wheel.targets == [], "nothing ran" + + +@pytest.mark.asyncio +async def test_a_check_carrying_several_properties_runs_once(tmp_path): + # One rule discharging three invariants is one check with three claims on it — three report + # rows, but a single thing to run and a single verdict. + wheel = _Wheel() + shared = _state(property_checks=[ + PropertyCheckMapping(property_title=title, checks=["c_stake"]) + for title in ("stake matches", "no double stake", "fees capped") + ]) + command = await _validate(wheel, tmp_path, state=shared) + assert [[c.name for c in t.checks] for t in wheel.targets] == [["c_stake"]] + assert set(command.update["verdicts"]) == {"c_stake"} + + +@pytest.mark.asyncio +async def test_the_stamping_run_records_what_it_covered(tmp_path): + # The publish gate validates the mapping against THIS run's checks, so a run that stamps has to + # say what it covered — and it says it as targets, so a component's coverage stays answerable + # even where a whole target erred. + command = await _validate(_Wheel(), tmp_path) + ran = command.update["ran"] + assert [t.name for t in ran] == ["c_farms", "c_fees"] + assert [[c.name for c in t.checks] for t in ran] == [["c_stake", "c_dbl"], ["c_fees"]] + + +# --------------------------------------------------------------------------- +# What the formalizer makes of a finished session +# --------------------------------------------------------------------------- + +@dataclass +class _Feat: + display_name: str = "Farms" + slug: str = "farms" + + def feature_json(self) -> dict[str, Any]: + return {"name": self.display_name, "slug": self.slug} + + +@dataclass +class _Source: + project_root: str + contract_name: str = "lending" + + +@dataclass +class _Ctx: + recursion_limit: int = 10 + + +@dataclass +class _Run: + source: _Source + ctx: _Ctx + env: None = None + + +@pytest.mark.asyncio +async def test_the_result_carries_the_targets_the_gating_run_covered(monkeypatch, tmp_path): + # The deliverable's sections are keyed by target name, so what the result claims as checked is + # what the stamping run covered — never what was declared, or what the properties suggest. + async def fake_session(**_kw): + return SessionResult( + commentary="done", + spec=SPEC, + skipped=[SkippedProperty(property_title="fees capped", reason="no oracle")], + property_checks=[("stake matches", ["c_stake"]), ("no double stake", ["c_dbl"])], + verdicts={}, + ran=[Target(name="c_farms", checks=[ + Check(name="c_stake", properties=["stake matches"], target="c_farms"), + Check(name="c_dbl", properties=["no double stake"], target="c_farms"), + ])], + expected_failures={}, + ) + + monkeypatch.setattr(adapter, "run_session", fake_session) + formalizer = adapter.RustFormalizer( + cast(Any, _Wheel()), AppDescriptor.model_validate(wire_descriptor()) + ) + props = [ + PropertyFormulation(title=m.property_title, sort="invariant", description="d") + for m in MAPPING + ] + result = await formalizer.formalize( + "Farms", cast(Any, _Feat()), props, cast(Any, _Ctx()), + cast(Any, _Run(_Source(str(tmp_path)), _Ctx())), cast(Any, None) + ) + + assert isinstance(result, adapter.RustFormalResult) + assert [t.name for t in result.targets] == ["c_farms"] + assert [c.name for c in result.targets[0].checks] == ["c_stake", "c_dbl"] + assert [s.property_title for s in result.skipped] == ["fees capped"] + assert dict(result.checks) == {"stake matches": ["c_stake"], "no double stake": ["c_dbl"]} diff --git a/tests/test_rustapp_verdicts.py b/tests/test_rustapp_verdicts.py new file mode 100644 index 00000000..c929eeda --- /dev/null +++ b/tests/test_rustapp_verdicts.py @@ -0,0 +1,117 @@ +"""Tests for the console/TUI verdict rollup (``composer/rustapp/results.py``). + +``report.json`` is the canonical results artifact, but it is written to disk — what a human watching +a run *sees* at the end is this rollup, so it has to account for every check that ran. The unit of +a verdict is a **unit**, not a component: ``units()`` is one unit per property, so a component with +five properties bakes five verdicts and owes five rows. + +Stubs throughout — no pipeline, no wheel. +""" + +import pathlib +from dataclasses import dataclass + +from composer.pipeline.core import CorePipelineResult, Delivered, GaveUp +from composer.pipeline.ptypes import ComponentOutcome +from composer.rustapp.result import RustFormalResult +from composer.rustapp.results import format_verdict_lines, summarize_verdicts +from composer.rustapp.wire import Verdict +from composer.spec.source.report.schema import Outcome + +#: Any report backend does — the point of these tests is that the rollup reads its wording out of +#: the report's own per-backend table rather than spelling outcomes itself. +BACKEND = "prover" + + +@dataclass +class _Feat: + """The one ``FeatureUnit`` member the rollup reads.""" + + display_name: str + slug: str = "component" + + +def _delivered(**verdicts: Outcome) -> Delivered[RustFormalResult]: + """A delivered component whose units check one property each, named ``_prop``.""" + return Delivered( + RustFormalResult( + checks=[(f"{name}_prop", [name]) for name in verdicts], + verdicts={unit: Verdict.with_outcome(outcome) for unit, outcome in verdicts.items()}, + ), + pathlib.Path("harness.rs"), + ) + + +def _result(*outcomes: ComponentOutcome) -> CorePipelineResult[RustFormalResult]: + return CorePipelineResult( + n_components=len(outcomes), n_properties=0, outcomes=list(outcomes), failures=[] + ) + + +def test_every_unit_of_a_component_gets_a_row(): + # A component with several properties bakes several verdicts. Reporting only the first would + # read as "1 Verified" where three checks ran, and would hide the failing one. + result = _result( + ComponentOutcome( + _Feat("increment"), [], + _delivered(rule_a=Outcome.GOOD, rule_b=Outcome.BAD, rule_c=Outcome.GOOD), + ) + ) + summary = summarize_verdicts(result, BACKEND) + + assert [(v.name, v.outcome) for v in summary.verdicts] == [ + ("rule_a_prop", Outcome.GOOD), + ("rule_b_prop", Outcome.BAD), + ("rule_c_prop", Outcome.GOOD), + ] + assert summary.counts == {Outcome.GOOD: 2, Outcome.BAD: 1} + assert summary.tally == "2 Verified, 1 Violated" + + +def test_rows_are_named_by_property_title_falling_back_to_the_unit(): + # The property's own words read better than the backend's unit name; a unit with no title in + # ``units`` (a wheel that reports a verdict for something it never declared) still gets a row. + delivered = _delivered(rule_a=Outcome.GOOD) + delivered.result.verdicts["orphan"] = Verdict.with_outcome(Outcome.ERROR) + summary = summarize_verdicts(_result(ComponentOutcome(_Feat("c"), [], delivered)), BACKEND) + + assert [v.name for v in summary.verdicts] == ["rule_a_prop", "orphan"] + + +def test_a_delivered_component_with_no_baked_verdict_is_still_listed(): + # A run-service-backed wheel reports through ``fetch_verdicts`` and bakes nothing; the component + # must not vanish from the listing. + result = _result( + ComponentOutcome(_Feat("no-verdicts"), [], Delivered(RustFormalResult(), pathlib.Path("h.rs"))) + ) + summary = summarize_verdicts(result, BACKEND) + + assert [(v.name, v.outcome) for v in summary.verdicts] == [("no-verdicts", Outcome.UNKNOWN)] + + +def test_give_ups_are_left_to_the_failures_block(): + result = _result( + ComponentOutcome(_Feat("gave-up"), [], GaveUp(reason="7 attempts")), + ComponentOutcome(_Feat("crashed"), [], RuntimeError("boom")), + ComponentOutcome(_Feat("ok"), [], _delivered(rule_a=Outcome.GOOD)), + ) + summary = summarize_verdicts(result, BACKEND) + + assert [v.name for v in summary.verdicts] == ["rule_a_prop"] + + +def test_the_listing_uses_the_reports_own_wording(): + result = _result( + ComponentOutcome(_Feat("c"), [], _delivered(rule_a=Outcome.GOOD, rule_b=Outcome.TIMEOUT)) + ) + lines = format_verdict_lines(summarize_verdicts(result, BACKEND)) + + assert lines[0] == " Verdicts: 1 Verified, 1 Timeout" + assert lines[1:] == [ + " ✓ rule_a_prop — Verified", + " ⧖ rule_b_prop — Timeout", + ] + + +def test_nothing_delivered_prints_nothing(): + assert format_verdict_lines(summarize_verdicts(_result(), BACKEND)) == [] diff --git a/tests/test_rustapp_wire.py b/tests/test_rustapp_wire.py new file mode 100644 index 00000000..212689e1 --- /dev/null +++ b/tests/test_rustapp_wire.py @@ -0,0 +1,269 @@ +"""The runtime ABI's Python side (``composer.rustapp.wire``). + +Every string crossing the FFI is one of these models, so this is where a shape mismatch with +``rust/autoprover-sdk/src/lib.rs`` should surface. Two properties matter: + +* the **tagged results** are discriminated unions — a build failure and a set of verdicts are + different types, and neither can be asked for the other's fields; +* the models are **strict**: both halves of the seam ship together, so a wheel that omits a field + or sends one the host doesn't declare has drifted, and either fails naming the field. Nothing here + reads an absent key as a default. + +No wheel needed — these are the payloads, not the callouts. For whether the two sides agree field by +field, see ``test_wire_roundtrip.py``, which checks it against the real serde types. +""" + +import json +import sys +import types + +import pytest +from pydantic import TypeAdapter, ValidationError + +from composer.rustapp.host import load_module +from composer.rustapp.wire import ( + CALLOUTS, + AuthorInput, + CompileFailed, + CompileOk, + ComponentInput, + PreflightInput, + SetupInput, + RustAppModule, + Check, + Target, + ValidateBuildFailed, + ValidateCoverageError, + ValidateVerdicts, + CalloutFailed, + expect_payload, + expect_text, + parse_compile, + parse_prompt, + parse_validate, + parse_workspace_prep, +) +from composer.spec.source.report.schema import Outcome +from tests.conftest import wire_verdict, wire_workspace_prep + + +def test_a_callout_error_is_not_read_as_a_payload(): + # The envelope is the one extra inbound shape: a host bug must not parse as a Prompt, an + # empty plan, or a failed build the author should revise. + with pytest.raises(CalloutFailed, match="invalid AuthorInput"): + parse_prompt('{"kind": "error", "message": "invalid AuthorInput JSON: eof"}') + with pytest.raises(CalloutFailed, match="boom"): + expect_text('{"kind": "error", "message": "boom"}') + assert expect_text(None) is None + assert expect_text("c_farms") == "c_farms" + assert expect_payload("Review the spec.") == "Review the spec." + + +def test_compile_result_is_discriminated_on_status(): + assert isinstance(parse_compile('{"status": "ok"}'), CompileOk) + failed = parse_compile('{"status": "failed", "errors": "E0432"}') + assert isinstance(failed, CompileFailed) and failed.errors == "E0432" + + +def test_validate_outcome_is_discriminated_on_kind(): + build_failed = parse_validate('{"kind": "build_failed", "errors": "no method `foo`"}') + assert isinstance(build_failed, ValidateBuildFailed) + + got = parse_validate(json.dumps( + {"kind": "verdicts", "verdicts": [["rule_a", wire_verdict("BAD", detail="cex")]]} + )) + assert isinstance(got, ValidateVerdicts) + unit, verdict = got.verdicts[0] + assert (unit, verdict.outcome, verdict.detail) == ("rule_a", Outcome.BAD, "cex") + + +def test_an_unknown_tag_is_refused_rather_than_read_as_the_other_variant(): + # A discriminated union, not a `kind == "build_failed"` test: an unrecognized tag has to be + # named here rather than fall through to the other variant's fields and die reading them. + with pytest.raises(ValidationError): + parse_validate('{"kind": "verdict", "verdicts": []}') + with pytest.raises(ValidationError): + parse_compile('{"status": "OK"}') + + +def test_an_omitted_field_is_refused_rather_than_read_as_a_default(): + # There is no wheel old enough to be excused one: the two halves ship together, so an absent + # `errors` is a mirror that drifted, and reading it as "" would put an empty revise prompt in + # front of the author instead of saying so. + with pytest.raises(ValidationError): + parse_compile('{"status": "failed"}') + with pytest.raises(ValidationError): + parse_validate('{"kind": "build_failed"}') + + +def test_a_field_the_host_does_not_declare_is_refused(): + # The other half of the same rule (`extra="forbid"` / `#[serde(deny_unknown_fields)]`): a key + # only one side knows means the mirrors disagree, and dropping it silently is how a wheel ends up + # reporting something no one reads. + with pytest.raises(ValidationError): + parse_compile('{"status": "ok", "warnings": ["unused"]}') + + +def test_a_null_target_means_a_check_is_its_own_validation_target(): + # `target` must be *present*; null is how the grouping says the check runs on its own. Absence + # is not a third spelling of that — see the test above. + own = Target.model_validate_json( + '{"name": "rule_p", "checks": [{"name": "rule_p", "properties": ["p"], "target": null}]}') + assert own.checks == [Check(name="rule_p", properties=["p"], target=None)] + assert own.checks[0].target_or_name() == "rule_p" + # …and a shared target is what the host runs once for several checks. + assert Check(name="rule_p", properties=["p"], target="c_vault").target_or_name() == "c_vault" + + +def test_an_outcome_the_host_does_not_know_is_refused(): + # The outcome vocabulary is closed on both sides, so a label the host has never heard of is a + # variant added to one `Outcome` and not the other — not something to render as UNKNOWN. + with pytest.raises(ValidationError): + parse_validate(json.dumps({"kind": "verdicts", + "verdicts": [["u", wire_verdict("FLAKY")]]})) + + +def _verdicts(*named: tuple[str, str]) -> ValidateVerdicts: + parsed = parse_validate(json.dumps( + {"kind": "verdicts", "verdicts": [[n, wire_verdict(o)] for n, o in named]})) + assert isinstance(parsed, ValidateVerdicts) + return parsed + + +_SHARED = Target(name="c_vault", checks=[ + Check(name="rule_p", properties=["p"], target="c_vault"), + Check(name="rule_q", properties=["q"], target="c_vault"), +]) + + +def test_a_verdict_resolves_to_the_check_the_host_sent(): + # The wire keys a verdict by name; upstream wants the `Check` the host sent, never one the + # wheel echoed back. Order is the target's, not the wheel's answer's. + resolved = _verdicts(("rule_q", "BAD"), ("rule_p", "GOOD")).resolve(_SHARED) + assert [(c.name, v.outcome) for c, v in resolved] == [ + ("rule_p", Outcome.GOOD), ("rule_q", Outcome.BAD), + ] + + +def test_a_check_left_without_a_verdict_is_refused(): + # The one that matters: an unanswered check is not a failing check, so nothing downstream has + # anything to object to — a wheel that answered for none of them would otherwise stamp the + # publish gate on a component it never checked. + with pytest.raises(ValidateCoverageError, match="rule_q"): + _verdicts(("rule_p", "GOOD")).resolve(_SHARED) + with pytest.raises(ValidateCoverageError): + _verdicts().resolve(_SHARED) + + +def test_a_verdict_for_a_check_the_target_does_not_cover_is_refused(): + # A name no check has can only be a wheel that invented one or misspelled one, and either way + # the verdict is about nothing the host can report under. + with pytest.raises(ValidateCoverageError, match="rule_r"): + _verdicts(("rule_p", "GOOD"), ("rule_q", "GOOD"), ("rule_r", "GOOD")).resolve(_SHARED) + + +def test_two_verdicts_for_one_check_are_refused_rather_than_one_winning(): + # Keying by name would quietly keep the last, which is how a BAD becomes a GOOD. + with pytest.raises(ValidateCoverageError, match="rule_p"): + _verdicts(("rule_p", "BAD"), ("rule_p", "GOOD"), ("rule_q", "GOOD")).resolve(_SHARED) + + +def test_a_workspace_plan_that_only_places_files_needs_no_toolchain(): + files_only = parse_workspace_prep(json.dumps( + wire_workspace_prep(files={"fuzz/Cargo.toml": "[package]"}))) + assert files_only.files and not files_only.needs_toolchain + # Whatever the request *says* is the chain's business; that there is one at all is the host's. + asking = parse_workspace_prep(json.dumps( + wire_workspace_prep(toolchain_request={"build_program": "lend"}))) + assert asking.needs_toolchain + + +def test_author_input_requires_what_the_wheel_requires(): + # `kind` selects the variant and `program` has no sensible default — omitting either is a host + # bug, and the wheel would otherwise be prompted about a program called "". + adapter = TypeAdapter(AuthorInput) + with pytest.raises(ValidationError): + adapter.validate_python({"program": "vault"}) + with pytest.raises(ValidationError): + adapter.validate_python({"kind": "component"}) + with pytest.raises(ValidationError): + adapter.validate_python({"kind": "not_a_kind", "program": "vault"}) + + +def test_each_author_kind_carries_only_its_own_payload(): + # The unit and the analyzed model belong to one kind each, so neither is a field the other two + # carry empty. A preflight has neither: it runs before anything is analyzed. + adapter = TypeAdapter(AuthorInput) + comp = adapter.validate_python( + {"kind": "component", "program": "vault", "unit": {"slug": "farms"}} + ) + assert isinstance(comp, ComponentInput) and comp.unit == {"slug": "farms"} + assert not hasattr(comp, "model") + setup = adapter.validate_python( + {"kind": "setup", "program": "vault", "model": {"components": []}} + ) + assert isinstance(setup, SetupInput) and setup.model == {"components": []} + assert not hasattr(setup, "unit") + pre = adapter.validate_python({"kind": "preflight", "program": "vault"}) + assert not hasattr(pre, "unit") and not hasattr(pre, "model") + + +def test_author_input_serializes_the_shape_the_wheel_deserializes(): + inp = PreflightInput(program="vault") + assert inp.model_dump() == { + "kind": "preflight", + "program": "vault", + # Empty rather than absent: the wheel is told the host resolved nothing and established + # nothing, and applies its own convention. Nothing here is filled in for it. + "source_unit": {}, + "props": [], + "setup": None, + "prep_facts": {}, + "args": {}, + } + + +def test_the_chain_shaped_payloads_cross_the_seam_uninterpreted(): + # source_unit / prep_facts / toolchain_request belong to the analyzed project's build system, and + # this side declares no schema for any of them: two chains with nothing in common go through + # unchanged. A field here would be the framework taking sides between them. + cargo = {"dir": "programs/lend", "package": "example-lending", "lib": "example_lending"} + move = {"package_dir": "sources", "named_addresses": {"vault": "0x1"}} + for unit in (cargo, move): + inp = TypeAdapter(AuthorInput).validate_json( + SetupInput(program="vault", source_unit=unit, prep_facts={"idl": "a.json"}) + .model_dump_json() + ) + assert inp.source_unit == unit and inp.prep_facts == {"idl": "a.json"} + + +def test_the_callout_list_is_derived_from_the_protocol(): + # The import-time check in `load_module` iterates this, so it must stay the protocol itself + # rather than a hand-kept copy of it. + assert set(CALLOUTS) == set(RustAppModule.__annotations__) + assert "workspace_prep" in CALLOUTS and "sandbox_grants" in CALLOUTS + + +def test_a_module_missing_callouts_is_refused_at_load(monkeypatch): + # `import_module` can't be checked statically, so the cast in `load_module` is guarded by this: + # a wheel built against an older SDK names its gaps here instead of dying with an AttributeError + # somewhere mid-run. + stub = types.ModuleType("not_a_wheel") + for name in CALLOUTS: + if name not in ("workspace_prep", "finalize"): + setattr(stub, name, lambda *_a: "{}") + monkeypatch.setitem(sys.modules, "not_a_wheel", stub) + + with pytest.raises(TypeError, match="workspace_prep, finalize"): + load_module("not_a_wheel") + + +def test_a_complete_module_loads(): + stub = types.ModuleType("a_wheel") + for name in CALLOUTS: + setattr(stub, name, lambda *_a: "{}") + sys.modules["a_wheel"] = stub + try: + assert load_module("a_wheel") is stub + finally: + del sys.modules["a_wheel"] diff --git a/tests/test_rustapp_workspace_prep.py b/tests/test_rustapp_workspace_prep.py new file mode 100644 index 00000000..2434fefa --- /dev/null +++ b/tests/test_rustapp_workspace_prep.py @@ -0,0 +1,165 @@ +"""Tests for the generic workspace prep and the per-chain seam under it +(``composer.rustapp.adapter.run_workspace_prep`` / ``composer.rustapp.toolchain``). + +The prep has two halves, and the split is the point: the host writes the plan's ``files`` itself +(that is the same in every ecosystem), and hands the plan's ``toolchain_request`` to the chain's +registered :class:`ProjectToolchain`, which is the only part that has to know what +``cargo-build-sbf`` or an IDL is. Everything crossing that seam is chain-shaped and opaque here — +these tests assert the framework *transports* it, never that it understands it. The ``source_unit`` +half of the same seam has the opposite failure mode: unregistered, it answers "nothing resolved" +rather than raising, because that is a state the wheel already handles. + +Fake wheel, fake chain — no toolchain, no LLM. +""" + +import json +from pathlib import Path +from typing import Any + +import pytest + +from composer.pipeline.ecosystem import EVM, SOLANA +from composer.rustapp.adapter import run_workspace_prep, source_unit_of +from composer.rustapp.toolchain import PROJECT_TOOLCHAINS +from composer.rustapp.wire import PreflightInput +from composer.spec.context import SourceFields +from composer.spec.system_model import SolidityIdentifier +from tests.conftest import wire_workspace_prep + +pytestmark = pytest.mark.asyncio + +#: What a registered toolchain reports about the analyzed project. Framework-side this is an opaque +#: object; that these keys spell a Cargo crate is the *chain's* business. +SOURCE_UNIT = {"dir": "programs/lend", "package": "example-lending", "lib": "example_lending"} +#: A chain-shaped prep request, and the facts carrying it out established. +REQUEST = {"warm_dirs": ["fuzz/vault"], "build_program": "example_lending", + "idl_dest": "fuzz/vault/idls/example_lending.json"} +PREP_FACTS = {"idl": "fuzz/vault/idls/example_lending.json"} + + +class FakeWheel: + """A wheel whose ``workspace_prep`` returns a fixed plan.""" + + def __init__(self, **plan): + self._plan = wire_workspace_prep(**plan) + + def workspace_prep(self, _input_json: str) -> str: + return json.dumps(self._plan) + + +class FakeToolchain: + """A chain implementation that records every call and reports fixed facts.""" + + def __init__(self, *, source_unit: dict[str, Any], prep_facts: dict[str, Any]): + self._source_unit = source_unit + self._prep_facts = prep_facts + self.calls: list[dict[str, Any]] = [] + + def source_unit(self, _source: SourceFields) -> dict[str, Any]: + return self._source_unit + + async def prepare(self, plan, input, *, source, sandbox, timeout_s) -> dict[str, Any]: + self.calls.append({ + "plan": plan, "input": input, "source": source, + "sandbox": sandbox, "timeout_s": timeout_s, + }) + return self._prep_facts + + +def _source(root: Path) -> SourceFields: + return SourceFields( + project_root=str(root), + contract_name=SolidityIdentifier("vault"), + relative_path="programs/lend/src/lib.rs", + forbidden_read="", + ) + + +def _register( + monkeypatch, *, source_unit: dict[str, Any] | None = None, + prep_facts: dict[str, Any] | None = None, +) -> FakeToolchain: + toolchain = FakeToolchain(source_unit=source_unit or {}, prep_facts=prep_facts or {}) + monkeypatch.setitem(PROJECT_TOOLCHAINS, "solana", toolchain) + return toolchain + + +async def _prep(wheel, root: Path, args: dict | None = None) -> dict[str, Any]: + return await run_workspace_prep( + wheel, + PreflightInput(program="vault", source_unit=SOURCE_UNIT, args=args or {}), + chain="solana", + source=_source(root), + sandbox=None, command_timeout_s=60, + ) + + +async def test_a_files_only_plan_is_complete_once_they_are_written(tmp_path, monkeypatch): + # An empty toolchain_request asks for nothing, so no toolchain is consulted. This is what makes + # an empty PROJECT_TOOLCHAINS a resting state rather than a broken one. + toolchain = _register(monkeypatch) + wheel = FakeWheel(files={"fuzz/vault/Cargo.toml": "[package]\n", "fuzz/vault/src/lib.rs": "//"}) + + assert await _prep(wheel, tmp_path) == {} + assert (tmp_path / "fuzz/vault/Cargo.toml").read_text() == "[package]\n" + assert (tmp_path / "fuzz/vault/src/lib.rs").read_text() == "//" + assert toolchain.calls == [] + + +async def test_wheel_written_files_are_path_confined(tmp_path, monkeypatch): + # The wheel is trusted, but its file paths go through the same confinement as everything else + # the host writes on its behalf — a traversal is refused rather than followed. + _register(monkeypatch) + with pytest.raises(ValueError, match="unsafe file path"): + await _prep(FakeWheel(files={"../../etc/evil": "x"}), tmp_path) + assert not (tmp_path.parent.parent / "etc" / "evil").exists() + + +async def test_the_toolchain_half_is_handed_the_plan_and_the_analyzed_source(tmp_path, monkeypatch): + toolchain = _register(monkeypatch, prep_facts=PREP_FACTS) + wheel = FakeWheel(files={"fuzz/vault/Cargo.toml": "[package]\n"}, toolchain_request=REQUEST) + + # Whatever the toolchain reports having established is what the caller reports back. + assert await _prep(wheel, tmp_path, {"program_idl": "/elsewhere/lend.json"}) == PREP_FACTS + assert len(toolchain.calls) == 1 + call = toolchain.calls[0] + # The request reaches it whole and uninterpreted — the host never reads a key of it, which is + # what lets a chain add one without touching composer.rustapp.wire. + assert call["plan"].toolchain_request == REQUEST + # The analyzed source, not just its root — an implementation resolves its own project facts from + # `relative_path` rather than being handed a shape the framework would have to understand. + assert call["source"].project_root == str(tmp_path) + assert call["source"].relative_path == "programs/lend/src/lib.rs" + # …and the whole AuthorInput, so it can read its own declared args (Crucible's `program_idl`) + # without the generic host having to know which flags mean anything. + assert call["input"].args["program_idl"] == "/elsewhere/lend.json" + assert call["timeout_s"] == 60 + # The files land before the toolchain runs: the manifest a warm or build reads is one of them. + assert (tmp_path / "fuzz/vault/Cargo.toml").is_file() + + +async def test_a_plan_needing_an_unregistered_toolchain_fails_loudly(tmp_path, monkeypatch): + # Skipping the preparation the wheel asked for would surface much later as a compile error the + # authoring agent cannot fix, so the seam refuses up front and names the fix. + monkeypatch.delitem(PROJECT_TOOLCHAINS, "solana", raising=False) + wheel = FakeWheel(toolchain_request={"build_program": "example_lending"}) + + with pytest.raises(ValueError, match="no project toolchain is registered"): + await _prep(wheel, tmp_path) + + +async def test_the_source_unit_comes_from_the_chains_toolchain(tmp_path, monkeypatch): + _register(monkeypatch, source_unit=SOURCE_UNIT) + assert source_unit_of(SOLANA, _source(tmp_path)) == SOURCE_UNIT + + +async def test_an_unresolved_source_unit_is_empty_rather_than_an_error(tmp_path, monkeypatch): + # Three different situations answer the same way — no toolchain for the chain, a language with no + # such unit (Solidity), and a layout the toolchain couldn't read — because the wheel does the + # same thing with all three: fall back to its own convention. + monkeypatch.delitem(PROJECT_TOOLCHAINS, "solana", raising=False) + assert source_unit_of(SOLANA, _source(tmp_path)) == {} + assert source_unit_of(EVM, _source(tmp_path)) == {} + + _register(monkeypatch, source_unit={}) + assert source_unit_of(SOLANA, _source(tmp_path)) == {} diff --git a/tests/test_sandbox_command.py b/tests/test_sandbox_command.py index 62a6453f..d503e234 100644 --- a/tests/test_sandbox_command.py +++ b/tests/test_sandbox_command.py @@ -4,9 +4,12 @@ trivial system binaries. Covers file materialization, path confinement, and the error/timeout paths. (It still backs the trusted Python build steps — e.g. the sBPF build; the Rust backend's own toolchain runs now go through ``run-confined`` in the -wheel, see ``docs/rust-backend-api.md``.) +wheel, see ``docs/rust-applications.md`` §8.) """ +import os +import time + import pytest from composer.sandbox.command import ( @@ -16,6 +19,17 @@ ) +def _pid_gone(pid: int, timeout_s: float = 2.0) -> bool: + deadline = time.monotonic() + timeout_s + while time.monotonic() < deadline: + try: + os.kill(pid, 0) + except ProcessLookupError: + return True + time.sleep(0.02) + return False + + @pytest.mark.asyncio async def test_run_local_command_materializes_files_and_captures_output(tmp_path): res = await run_local_command( @@ -62,3 +76,19 @@ async def test_run_local_command_timeout(tmp_path): res = await run_local_command("sleep", ["5"], {}, workdir=tmp_path, timeout_s=1) assert res.exit_code == -1 assert "timed out" in res.stderr + + +@pytest.mark.asyncio +async def test_run_local_command_timeout_kills_process_group(tmp_path): + """A forked grandchild must die with the timed-out leader (process-group SIGKILL).""" + res = await run_local_command( + "sh", + ["-c", "sleep 100 & echo $! > child.pid; exec sleep 100"], + {}, + workdir=tmp_path, + timeout_s=1, + ) + assert res.exit_code == -1 + assert "timed out" in res.stderr + grandchild = int((tmp_path / "child.pid").read_text().strip()) + assert _pid_gone(grandchild), f"grandchild {grandchild} still alive after timeout" diff --git a/tests/test_sandbox_config.py b/tests/test_sandbox_config.py index a23e79b9..897bf471 100644 --- a/tests/test_sandbox_config.py +++ b/tests/test_sandbox_config.py @@ -122,17 +122,20 @@ def test_rust_build_policy_shape(tmp_path, monkeypatch): def test_rust_build_policy_offline_sets_cargo_net_offline(tmp_path): - """Default (offline) forces every cargo — incl. the one `crucible run` spawns — - offline via CARGO_NET_OFFLINE; opting out drops it.""" + """Default (offline) forces every cargo — incl. a nested one a checker spawns — offline via + CARGO_NET_OFFLINE; opting out drops it. + + The value has to be exactly `true`: cargo parses this as a config boolean and refuses + anything else, so `1` fails the build *and* leaves it online.""" on = rust_build_policy(tmp_path) - assert on.env_allowlist.get("CARGO_NET_OFFLINE") == "1" + assert on.env_allowlist.get("CARGO_NET_OFFLINE") == "true" off = rust_build_policy(tmp_path, offline=False) assert "CARGO_NET_OFFLINE" not in off.env_allowlist def test_config_enabled_policy_is_offline_by_default(tmp_path): pol = SandboxConfig(provider="launcher").build_policy(tmp_path) - assert pol.env_allowlist.get("CARGO_NET_OFFLINE") == "1" + assert pol.env_allowlist.get("CARGO_NET_OFFLINE") == "true" pol_net = SandboxConfig(provider="launcher", offline=False).build_policy(tmp_path) assert "CARGO_NET_OFFLINE" not in pol_net.env_allowlist diff --git a/tests/test_sandbox_escape.py b/tests/test_sandbox_escape.py index 21c63a39..1c2d600b 100644 --- a/tests/test_sandbox_escape.py +++ b/tests/test_sandbox_escape.py @@ -16,8 +16,8 @@ Runnable without the full Crucible stack (std-only program, no crates, no network needed to compile). Skipped unless `rustc` and a working launcher are present. The -*legitimate* half (a real `solana_vault` build+fuzz under the launcher) is the -expensive Part B in `tests/test_crucible_sandbox_gate.py`. +*legitimate* half (a real program build+fuzz under the launcher) is the expensive +Part B, which ships with the Crucible backend. """ import asyncio diff --git a/tests/test_sandbox_launcher.py b/tests/test_sandbox_launcher.py index a15c035b..8d3f68e9 100644 --- a/tests/test_sandbox_launcher.py +++ b/tests/test_sandbox_launcher.py @@ -2,7 +2,7 @@ The ``wrap`` tests are pure argv construction (no binary, no subprocess) and pin the exact flag mapping. The ``available`` / ``--probe`` tests exercise the real -binary when it has been built (``cargo build -p run-confined --release``) and skip +binary when it has been built (``uv sync`` installs it into ``.venv/bin``) and skip otherwise, so the suite stays green on a machine without the Rust build. """ diff --git a/tests/test_sandbox_run_confined.py b/tests/test_sandbox_run_confined.py index 09e1ad3a..7fcd0551 100644 --- a/tests/test_sandbox_run_confined.py +++ b/tests/test_sandbox_run_confined.py @@ -9,6 +9,7 @@ import asyncio import os +import time from pathlib import Path import pytest @@ -141,6 +142,35 @@ async def test_none_provider_is_not_confined(tmp_path): assert "readable" in res.stdout +def _pid_gone(pid: int, timeout_s: float = 2.0) -> bool: + deadline = time.monotonic() + timeout_s + while time.monotonic() < deadline: + try: + os.kill(pid, 0) + except ProcessLookupError: + return True + time.sleep(0.02) + return False + + +@_needs_sandbox +async def test_confined_timeout_kills_process_group(tmp_path): + """``run-confined`` execve's in place; the timeout must still kill fork children.""" + res = await run_local_command( + "sh", + ["-c", "sleep 100 & echo $! > child.pid; exec sleep 100"], + {}, + workdir=tmp_path, + timeout_s=1, + provider=_PROVIDER, + policy=_system_policy(tmp_path), + ) + assert res.exit_code == -1, res.stderr + assert "timed out" in res.stderr + grandchild = int((tmp_path / "child.pid").read_text().strip()) + assert _pid_gone(grandchild), f"grandchild {grandchild} still alive after timeout" + + async def test_unavailable_provider_fails_closed(tmp_path): """A provider that reports unavailable must raise, never run unconfined.""" diff --git a/tests/test_solana_component_grouping.py b/tests/test_solana_component_grouping.py new file mode 100644 index 00000000..52bcfbd9 --- /dev/null +++ b/tests/test_solana_component_grouping.py @@ -0,0 +1,226 @@ +"""Probe: run ONLY the Solana system-analysis phase against a real program and print the +``ProgramComponent`` grouping it produces. + +It stops after analysis (no property extraction, no backend), so it costs one agent run rather +than a full front half, and it exercises the model, the prompt section, and +``_solana_validate``'s component rules. + +Points at a program via environment, and skips when unset (this repo carries no Solana program +large enough for its grouping to prove anything): + + SOLANA_PROBE_ROOT=/path/to/workspace \\ + SOLANA_PROBE_DOC=docs/DESIGN.md \\ + SOLANA_PROBE_SRC=programs//src/lib.rs \\ + SOLANA_PROBE_MAIN= \\ + env -u CERTORA .venv/bin/python -m pytest tests/test_solana_component_grouping.py \\ + -m expensive -q -s +""" + +import asyncio +import os +from pathlib import Path +from types import SimpleNamespace +from typing import Any, TYPE_CHECKING, cast + +import psycopg +import pytest +from psycopg.sql import SQL, Identifier, Literal + +import composer.workflow.services as services +from composer.io.multi_job import TaskInfo +from composer.rag.models import DefaultEmbedder +from composer.llm.registry import get_provider_for +from composer.pipeline.core import PipelineRun +from composer.pipeline.ecosystem import RUST_FORBIDDEN_READ, SOLANA +from composer.rustapp.frontend import GenericRustConsoleHandler +from composer.spec.context import CacheKey, SourceCode, WorkflowContext +from composer.spec.service_host import ModelProvider, PureServiceHost +from composer.spec.solana.model import SolanaApplication +from composer.spec.solana.null_backend import SolanaPhase +from composer.spec.source.source_env import build_basic_source_tools, build_source_tools +from composer.spec.source.task_ids import SYSTEM_ANALYSIS_TASK_ID +from composer.spec.system_analysis import run_component_analysis +from composer.spec.system_model import SolidityIdentifier +from composer.ui.tool_display import async_tool_context +from composer.workflow.services import standard_connections + +from tests.conftest import ( + _MEMORIES_DDL, + _RAG_DB, + _VECTOR_DBS, + MockSentenceTransformer, + _db_url, + needs_postgres, +) + +if TYPE_CHECKING: + from testcontainers.postgres import PostgresContainer + +pytestmark = [pytest.mark.expensive, needs_postgres, pytest.mark.asyncio] + + +def _target() -> tuple[Path, Path, str, str] | None: + root = os.environ.get("SOLANA_PROBE_ROOT") + if not root: + return None + return ( + Path(root), + Path(root) / os.environ["SOLANA_PROBE_DOC"], + os.environ["SOLANA_PROBE_SRC"], + os.environ["SOLANA_PROBE_MAIN"], + ) + + +def _model_args() -> object: + return SimpleNamespace( + heavy_model="claude-opus-4-6", + lite_model="claude-sonnet-4-6", + tokens=128_000, + thinking_tokens=8192, + memory_tool=False, + interleaved_thinking=False, + ) + + +def _provision(pg_container: "PostgresContainer") -> None: + """The roles/databases the pipeline expects (same as the solana gate).""" + admin_url = pg_container.get_connection_url(driver=None) + with psycopg.connect(admin_url, autocommit=True) as admin: + for cfg in services._DATABASE_CONFIGS.values(): + admin.execute( + SQL("CREATE ROLE {} LOGIN PASSWORD {}").format( + Identifier(cfg["user"]), Literal(cfg["password"]) + ) + ) + admin.execute( + SQL("CREATE DATABASE {} OWNER {}").format( + Identifier(cfg["database"]), Identifier(cfg["user"]) + ) + ) + admin.execute(SQL("CREATE DATABASE {}").format(Identifier(_RAG_DB))) + for db in _VECTOR_DBS: + with psycopg.connect(_db_url(pg_container, db), autocommit=True) as conn: + conn.execute("CREATE EXTENSION IF NOT EXISTS vector") + # As the memory ROLE, not the superuser — the table must be owned by the role that will read + # it, or the agent's first memory-tool call dies on "permission denied for table memories_fs". + mem = services._DATABASE_CONFIGS["memory"] + mem_url = ( + f"postgresql://{mem['user']}:{mem['password']}" + f"@{pg_container.get_container_host_ip()}:{pg_container.get_exposed_port(5432)}" + f"/{mem['database']}" + ) + with psycopg.connect(mem_url, autocommit=True) as conn: + conn.execute(_MEMORIES_DDL) + + +def _report(app: SolanaApplication, main: str) -> None: + print(f"\n=== {app.application_type}: {app.description}") + print(f" authorities: {', '.join(a.name for a in app.authorities) or '(none)'}") + for prog in app.programs: + star = " <== main" if prog.program_identifier == main else "" + print(f"\n--- program {prog.name} ({prog.program_identifier}){star}") + print(f" {len(prog.instructions)} instructions, {len(prog.components)} components") + assigned: set[str] = set() + for comp in prog.components: + assigned.update(comp.instructions) + print(f"\n [{comp.name}] {comp.description}") + print(f" instructions ({len(comp.instructions)}): {', '.join(comp.instructions)}") + if comp.account_types: + print(f" state: {', '.join(comp.account_types)}") + for req in comp.requirements: + print(f" req: {req}") + for inter in comp.interactions: + target = getattr(inter, "authority", None) or ( + f"{inter.program}.{inter.component}" # type: ignore[union-attr] + ) + print(f" -> {target}: {inter.description}") + # The validator already enforces this; print it so a human can see the shape at a glance. + overlap = [i.name for i in prog.instructions if sum( + i.name in c.instructions for c in prog.components) > 1] + print(f"\n coverage: {len(assigned)}/{len(prog.instructions)} instructions assigned" + f"{f'; {len(overlap)} in >1 component: {overlap}' if overlap else ''}") + + +async def test_component_grouping_on_a_real_program(pg_container: "PostgresContainer", monkeypatch): + target = _target() + if target is None: + pytest.skip("set SOLANA_PROBE_ROOT/_DOC/_SRC/_MAIN to point at a real Solana workspace") + root, doc, src_rel, main_id = target + assert root.is_dir() and doc.is_file(), (root, doc) + + _provision(pg_container) + monkeypatch.setenv("CERTORA_AI_COMPOSER_PGHOST", pg_container.get_container_host_ip()) + monkeypatch.setenv("CERTORA_AI_COMPOSER_PGPORT", str(pg_container.get_exposed_port(5432))) + + args = _model_args() + root_s = str(root) + # Built BEFORE the connections: `standard_connections` takes the resolved provider + # service (it asks it for an uploader and the memory tool), not a provider name. + tiered = get_provider_for(tiered=cast(Any, args)) + async with ( + standard_connections( + provider=tiered.provider_service, + embedder=DefaultEmbedder(MockSentenceTransformer()), + ) as conns, + async_tool_context(), + ): + content = await conns.uploader.get_document(doc) + assert content is not None + source = SourceCode( + content=content, + project_root=root_s, + contract_name=SolidityIdentifier(main_id), + relative_path=src_rel, + forbidden_read=RUST_FORBIDDEN_READ, + ) + models = ModelProvider( + heavy_model=tiered.heavy, lite_model=tiered.lite, checkpointer=conns.checkpointer + ) + basic = build_basic_source_tools(root=root_s, forbidden_read=RUST_FORBIDDEN_READ) + full = build_source_tools( + basic, models, conns.indexed_store, ("solana_grouping", "src"), recursion_limit=100 + ) + env = PureServiceHost(models=models, rag_tools=(), sort="existing").bind_source_tools(full) + ctx: WorkflowContext[Any] = WorkflowContext.create( + services=conns.memory, + thread_id="solana_grouping", + store=conns.store, + recursion_limit=100, + cache_namespace=None, + memory_namespace=None, + ) + + # Go through PipelineRun.runner so the IO handler / task scope the analysis graph needs is + # installed — the same path run_pipeline takes, minus every phase after analysis. + run = PipelineRun( + ctx=ctx, + source=source, + _handler_factory=GenericRustConsoleHandler(set()).make_handler, + _agent_semaphore=asyncio.Semaphore(4), + _cpu_semaphore=asyncio.Semaphore(2), + env=env, + ) + analyzed = await run.runner( + TaskInfo(SYSTEM_ANALYSIS_TASK_ID, "System Analysis", SolanaPhase.ANALYSIS), + lambda: run_component_analysis( + ty=SOLANA.system_model, + child_ctxt=ctx.child(CacheKey("solana-analysis")), + input=source, + env=env, + extra_input=list(SOLANA.analysis_extra_input(source)), + expected_main_id=source.contract_name, + system_template=SOLANA.analysis_prompts.system, + initial_template=SOLANA.analysis_prompts.initial, + validate=SOLANA.validate_analysis, + ), + ) + + assert analyzed is not None, "system analysis produced no result" + assert isinstance(analyzed, SolanaApplication) + _report(analyzed, main_id) + + # Stage 1's contract: analysis emits components, and they satisfy the validator (which the + # analysis loop already enforced via retry — re-asserted here so a regression is loud). + main = next(p for p in analyzed.programs if p.program_identifier == main_id) + assert main.components, "the main program came back with no components" + assert SOLANA.validate_analysis(analyzed, source.contract_name) is None diff --git a/tests/test_solc_version_resolver.py b/tests/test_solc_version_resolver.py index a2bcf06c..9de33850 100644 --- a/tests/test_solc_version_resolver.py +++ b/tests/test_solc_version_resolver.py @@ -45,3 +45,18 @@ def test_exact_pragma_ignores_installed_state(monkeypatch) -> None: def test_preferred_version_still_wins(monkeypatch) -> None: _with_installed(monkeypatch, {"solc8.35"}) assert svr.resolve_pragma_to_version("^0.8.0", preferred_version="solc8.28") == "0.8.28" + + +def test_pragma_admits_reports_true_false_or_unknown() -> None: + # An exact pin admits only itself, whichever way it is spelled. + assert svr.pragma_admits("0.6.4", "0.6.4") is True + assert svr.pragma_admits("0.6.4", "0.8.34") is False + assert svr.pragma_admits("=0.6.4", "0.8.34") is False + # Ranges admit what they cover. + assert svr.pragma_admits("^0.8.0", "0.8.34") is True + assert svr.pragma_admits("~0.6.4", "0.6.12") is True + assert svr.pragma_admits("~0.6.4", "0.7.0") is False + assert svr.pragma_admits(">=0.4.22 <0.9.0", "0.5.16") is True + # A disjunction is not expressible as one SpecifierSet, so it is unknown rather + # than a contradiction — callers must not read None as "rejected". + assert svr.pragma_admits("^0.6.0 || ^0.8.0", "0.8.34") is None diff --git a/tests/test_soroban_components.py b/tests/test_soroban_components.py new file mode 100644 index 00000000..483128d6 --- /dev/null +++ b/tests/test_soroban_components.py @@ -0,0 +1,298 @@ +from typing import Any, cast + +import pytest +from pydantic import ValidationError + +from composer.pipeline.ecosystem import SOROBAN, _soroban_validate +from composer.spec.soroban.model import ( + AuthorityInteraction, + InterComponentInteraction, + SorobanApplication, + SorobanContractInstance, +) +from composer.spec.types import RustIdentifier + +VAULT_ID = RustIdentifier("vault") + + +def _raw() -> dict[str, Any]: + return { + "application_type": "Vault", + "description": "A single-contract token vault.", + "components": [ + { + "name": "Vault", + "contract_identifier": VAULT_ID, + "description": "Holds per-address deposits.", + "storage_entries": [ + { + "key": "Balance(Address)", + "durability": "persistent", + "value_type": "i128", + "description": "per-depositor balance", + }, + { + "key": "Admin", + "durability": "instance", + "value_type": "Address", + "description": "the address allowed to upgrade", + }, + ], + "functions": [ + { + "name": "initialize", + "description": "record the admin", + "requirements": [], + }, + { + "name": "deposit", + "description": "move tokens in", + "auth": [ + { + "address": "from", + "kind": "require_auth", + "description": "the depositor must authorize", + } + ], + "requirements": [], + }, + {"name": "withdraw", "description": "move tokens out", "requirements": []}, + ], + "components": [ + { + "name": "Deposits", + "description": "Initializing the vault and funding it.", + "functions": ["initialize", "deposit"], + "storage_keys": ["Balance(Address)", "Admin"], + "interactions": [ + {"authority": "Token", "description": "calls transfer in"} + ], + "requirements": ["The implementation must credit the depositor."], + }, + { + "name": "Withdrawals", + "description": "Releasing funds to the recorded depositor.", + "functions": ["withdraw"], + "storage_keys": ["Balance(Address)"], + "interactions": [ + { + "contract": "Vault", + "component": "Deposits", + "description": "reads the balance Deposits maintains", + } + ], + "requirements": ["The implementation must only pay the depositor."], + }, + ], + }, + { + "name": "Token", + "description": "A SEP-41 token the vault holds.", + "assumptions": ["A Stellar Asset Contract; the issuer may clawback."], + }, + ], + } + + +def _app(mutate=None) -> SorobanApplication: + raw = _raw() + if mutate is not None: + mutate(raw) + return SorobanApplication.model_validate(raw) + + +def _contract(raw: dict[str, Any]) -> dict[str, Any]: + return raw["components"][0] + + +def _components(raw: dict[str, Any]) -> list[dict[str, Any]]: + return _contract(raw)["components"] + + +def test_components_parse_and_functions_resolve_through_the_contract(): + contract = _app().contracts[0] + assert [c.name for c in contract.components] == ["Deposits", "Withdrawals"] + deposits = contract.components[0] + resolved = [contract.functions_by_name[n] for n in deposits.functions] + assert [f.description for f in resolved] == ["record the admin", "move tokens in"] + + +def test_absent_auth_is_recorded_as_empty_not_missing(): + by_name = _app().contracts[0].functions_by_name + assert by_name["withdraw"].auth == [] + assert [a.address for a in by_name["deposit"].auth] == ["from"] + + +def test_to_signature_renders_args_and_return(): + """The join/`->` assembly lives on the model, not in `component_context.j2`, so it is + testable without rendering a template.""" + def typed(raw): + fns = _contract(raw)["functions"] + fns[1]["args"] = ["from: Address", "to: Address", "amount: i128"] + fns[1]["returns"] = "Result<(), Error>" + + by_name = _app(typed).contracts[0].functions_by_name + assert ( + by_name["deposit"].to_signature() + == "deposit(from: Address, to: Address, amount: i128) -> Result<(), Error>" + ) + assert by_name["initialize"].to_signature() == "initialize()" + + +def test_interaction_union_discriminates_by_shape(): + contract = _app().contracts[0] + assert isinstance(contract.components[0].interactions[0], AuthorityInteraction) + inter = contract.components[1].interactions[0] + assert isinstance(inter, InterComponentInteraction) + assert (inter.contract, inter.component) == ("Vault", "Deposits") + + +def test_unit_resolves_functions_and_durability_tagged_storage(): + (deposits, withdrawals) = SOROBAN.units(SorobanContractInstance(0, _app())) + assert deposits.display_name == "Deposits" and deposits.slug == "Deposits" + assert [f.name for f in withdrawals.functions] == ["withdraw"] + assert [(e.key, e.durability) for e in deposits.storage_entries] == [ + ("Balance(Address)", "persistent"), + ("Admin", "instance"), + ] + feature = deposits.feature_json() + assert feature["slug"] == "Deposits" + functions = cast(list[dict[str, Any]], feature["functions"]) + storage = cast(list[dict[str, Any]], feature["storage_entries"]) + assert [f["name"] for f in functions] == ["initialize", "deposit"] + assert [e["durability"] for e in storage] == ["persistent", "instance"] + + +def test_well_formed_application_validates(): + assert _soroban_validate(_app(), VAULT_ID) is None + + +def test_expected_main_is_required(): + problem = _soroban_validate(_app(), RustIdentifier("not_the_vault")) + assert problem is not None and "not_the_vault" in problem + + +def test_duplicate_component_names_rejected(): + def dup(raw): + _components(raw)[1]["name"] = "Deposits" + + problem = _soroban_validate(_app(dup), VAULT_ID) + assert problem is not None and "Duplicate component names in Vault: Deposits" in problem + + +def test_component_slug_collision_rejected(): + def collide(raw): + _components(raw)[1]["name"] = "Deposits!" + + problem = _soroban_validate(_app(collide), VAULT_ID) + assert problem is not None and "filename slug" in problem + + +def test_unknown_function_reference_rejected(): + def typo(raw): + _components(raw)[0]["functions"] = ["initialize", "depsoit"] + + problem = _soroban_validate(_app(typo), VAULT_ID) + assert problem is not None + assert "lists a function 'depsoit' that Vault does not declare" in problem + + +def test_function_belonging_to_no_component_rejected(): + def orphan(raw): + _components(raw)[0]["functions"] = ["initialize"] # drops `deposit` + + problem = _soroban_validate(_app(orphan), VAULT_ID) + assert problem is not None and "'deposit'" in problem and "belong to no component" in problem + + +def test_a_function_may_serve_two_components(): + def overlap(raw): + _components(raw)[1]["functions"] = ["withdraw", "initialize"] + + assert _soroban_validate(_app(overlap), VAULT_ID) is None + + +def test_a_contract_with_no_functions_needs_no_components(): + def empty(raw): + _contract(raw)["functions"] = [] + _contract(raw)["components"] = [] + + assert _soroban_validate(_app(empty), VAULT_ID) is None + + +@pytest.mark.parametrize( + "interaction, expected", + [ + ({"authority": "Reflector", "description": "reads a price"}, + "unknown external authority: Reflector"), + ( + {"contract": "Pool", "component": "Swaps", "description": "x"}, + "an unknown contract: Pool", + ), + ( + {"contract": "Vault", "component": "Rewards", "description": "x"}, + "unknown component Rewards of contract Vault", + ), + ], +) +def test_unresolvable_interactions_rejected(interaction, expected): + def point_nowhere(raw): + _components(raw)[0]["interactions"] = [interaction] + + problem = _soroban_validate(_app(point_nowhere), VAULT_ID) + assert problem is not None and expected in problem + + +def test_interaction_component_is_required(): + """Unlike EVM's ``ComponentInteraction`` and Solana's peer, which allow a null component: + analysis authors the callee's components in the same response as the interaction, so there is + no point at which they are unknown. + + Asserted against the model directly rather than through a whole application: an application + whose contract fails to parse does not raise, because ``SorobanContract | SorobanAuthority`` + falls back to the authority arm (it needs only ``name`` + ``description``, which a contract + dict also has).""" + ok = {"contract": "Vault", "component": "Deposits", "description": "x"} + assert InterComponentInteraction.model_validate(ok).component == "Deposits" + with pytest.raises(ValidationError): + InterComponentInteraction.model_validate({**ok, "component": None}) + + +def test_unknown_storage_key_reference_rejected(): + def typo(raw): + _components(raw)[1]["storage_keys"] = ["Blance(Address)"] + + problem = _soroban_validate(_app(typo), VAULT_ID) + assert problem is not None + assert "lists a storage key 'Blance(Address)'" in problem + + +def test_a_key_declared_under_two_durabilities_rejected(): + def shadow(raw): + _contract(raw)["storage_entries"].append( + { + "key": "Balance(Address)", + "durability": "temporary", + "value_type": "i128", + "description": "a cache of the same balance", + } + ) + + problem = _soroban_validate(_app(shadow), VAULT_ID) + assert problem is not None + assert "declared twice in Vault" in problem and "separate key spaces" in problem + + +def test_a_component_need_not_claim_every_storage_key(): + def drop(raw): + _components(raw)[0]["storage_keys"] = [] + + assert _soroban_validate(_app(drop), VAULT_ID) is None + + +def test_soroban_is_registered_and_locates_its_main(): + from composer.pipeline.ecosystem import ECOSYSTEMS + + assert ECOSYSTEMS["soroban"] is SOROBAN + assert SOROBAN.name == "soroban" and SOROBAN.language.name == "rust" + assert not SOROBAN.supports_greenfield diff --git a/tests/test_stuck_rule_warnings.py b/tests/test_stuck_rule_warnings.py new file mode 100644 index 00000000..797cc454 --- /dev/null +++ b/tests/test_stuck_rule_warnings.py @@ -0,0 +1,169 @@ +"""Regression tests for :func:`stuck_rule_warnings` — the prover author's "you have been +stuck on this rule for three runs" detector. + +The loop this covers used to be inline in ``verify_spec`` and carried three defects: + +1. it iterated ``stuck_count.keys()`` while deleting from it, so any run that broke a + rule's streak crashed the whole CVL-generation task with ``RuntimeError: dictionary + changed size during iteration``; +2. it never advanced the history index on the ``run`` branch, so it re-counted the *same* + prover run until the threshold tripped — every first stuck run nagged as though it had + already failed identically three times; +3. it did ``del stuck_count[r]`` for previously-nagged rules, which ``KeyError``s whenever + a rule that was nagged before isn't stuck now. +""" + +from composer.prover.ptypes import RulePath +from composer.spec.source.prover import ( + NagMarker, ProverHistoryItem, ProverRunLog, RuleSelection, STUCK_RULE_NAG_THRESHOLD, + stuck_rule_warnings, +) + +R1 = RulePath(rule="r1") +R2 = RulePath(rule="r2") + + +def _run( + *results: tuple[RulePath, str], + tc_id: str = "tc", + rules: RuleSelection | None = None, + declared: tuple[str, ...] = ("r1", "r2"), +) -> ProverHistoryItem: + return ProverRunLog( + tool_call_id=tc_id, + prover_results=list(results), # type: ignore[arg-type] + rules=rules, + spec_digest="d", + sort="run", + declared_rules=list(declared), + state_digest="sd", + ) + + +def _warn(stuck, history, known=("tc",)): + return stuck_rule_warnings(stuck, history, set(known)) + + +def test_threshold_counts_the_current_run_plus_two_priors() -> None: + # Defect 2: a single stuck run must NOT nag. The tally starts at 1 for the run being + # processed, so it takes two prior identical failures to reach the threshold. + assert STUCK_RULE_NAG_THRESHOLD == 3 + + stuck = {R1: "TIMEOUT"} + assert _warn(stuck, [])[0] == set() + assert _warn(stuck, [_run((R1, "TIMEOUT"))])[0] == set() + assert _warn(stuck, [_run((R1, "TIMEOUT")), _run((R1, "TIMEOUT"))])[0] == {R1} + + +def test_a_broken_streak_drops_the_rule() -> None: + # Defect 1: this is the shape that used to raise RuntimeError — a rule leaves the + # tally mid-iteration because an earlier run did not fail identically. + stuck = {R1: "TIMEOUT"} + history = [ + _run((R1, "VIOLATED")), # oldest: different status, breaks the streak + _run((R1, "TIMEOUT")), + ] + assert _warn(stuck, history)[0] == set() + + # A differing status anywhere in the walk-back stops the count, even with plenty of + # older identical failures behind it. + history = [ + _run((R1, "TIMEOUT")), _run((R1, "TIMEOUT")), _run((R1, "TIMEOUT")), + _run((R1, "VIOLATED")), + _run((R1, "TIMEOUT")), + ] + assert _warn(stuck, history)[0] == set() + + +def test_rules_are_tallied_independently() -> None: + # Exercises the delete-during-iteration path with a survivor alongside it. + stuck = {R1: "TIMEOUT", R2: "ERROR"} + history = [ + _run((R1, "TIMEOUT"), (R2, "VIOLATED")), + _run((R1, "TIMEOUT"), (R2, "ERROR")), + ] + assert _warn(stuck, history)[0] == {R1} + + +def test_targeted_runs_are_transparent_to_untouched_rules() -> None: + # A re-run scoped to r2 neither extends nor breaks r1's streak. + stuck = {R1: "TIMEOUT"} + history = [ + _run((R1, "TIMEOUT")), + _run((R2, "ERROR"), rules={"sort": "include", "selector": ["r2"]}), + _run((R1, "TIMEOUT")), + ] + assert _warn(stuck, history)[0] == {R1} + + +def test_exclude_scoped_runs_are_transparent_to_excluded_rules() -> None: + # A run that excluded r1 never exercised it, so it neither extends nor breaks + # r1's streak — the identical failures on either side of it still add up. + stuck = {R1: "TIMEOUT"} + history = [ + _run((R1, "TIMEOUT")), + _run((R2, "ERROR"), rules={"sort": "exclude", "selector": ["r1"]}), + _run((R1, "TIMEOUT")), + ] + assert _warn(stuck, history)[0] == {R1} + + +def test_exclude_scoped_run_that_covers_the_rule_breaks_its_streak() -> None: + # An exclude-scoped run still executes every non-excluded declared rule, so r1 + # passing in it ends the streak like any full run would. + stuck = {R1: "TIMEOUT"} + history = [ + _run((R1, "TIMEOUT")), + _run((R1, "VERIFIED"), rules={"sort": "exclude", "selector": ["r2"]}), + _run((R1, "TIMEOUT")), + ] + assert _warn(stuck, history)[0] == set() + + +def test_full_runs_scope_to_their_declared_rules() -> None: + # A full run from before r1 was declared is transparent to r1's streak: "all + # rules" means the rules that run declared, not today's. + stuck = {R1: "TIMEOUT"} + history = [ + _run((R1, "TIMEOUT")), + _run((R2, "VERIFIED"), declared=("r2",)), + _run((R1, "TIMEOUT")), + ] + assert _warn(stuck, history)[0] == {R1} + + +def test_nag_marker_restarts_the_streak() -> None: + stuck = {R1: "TIMEOUT"} + history = [ + _run((R1, "TIMEOUT")), + _run((R1, "TIMEOUT")), + NagMarker(nagged_rules=[R1], sort="nag"), + _run((R1, "TIMEOUT")), + ] + # Without the marker this would be four identical failures; the marker means R1 was + # already warned about, so the author isn't nagged again for the same stretch. + assert _warn(stuck, history)[0] == set() + + +def test_nag_marker_for_a_rule_that_is_no_longer_stuck() -> None: + # Defect 3: R2 was nagged previously but only R1 is stuck now — must not KeyError. + stuck = {R1: "TIMEOUT"} + history = [ + _run((R1, "TIMEOUT")), + NagMarker(nagged_rules=[R2], sort="nag"), + _run((R1, "TIMEOUT")), + ] + assert _warn(stuck, history)[0] == {R1} + + +def test_reports_history_older_than_the_last_compaction() -> None: + stuck = {R1: "TIMEOUT"} + visible = [_run((R1, "TIMEOUT"), tc_id="tc"), _run((R1, "TIMEOUT"), tc_id="tc")] + assert _warn(stuck, visible, known=("tc",)) == ({R1}, False) + + compacted = [_run((R1, "TIMEOUT"), tc_id="gone"), _run((R1, "TIMEOUT"), tc_id="tc")] + assert _warn(stuck, compacted, known=("tc",)) == ({R1}, True) + + +def test_no_stuck_rules_is_a_no_op() -> None: + assert _warn({}, [_run((R1, "TIMEOUT"))]) == (set(), False) diff --git a/tests/test_task_budgets.py b/tests/test_task_budgets.py new file mode 100644 index 00000000..8e65e6cc --- /dev/null +++ b/tests/test_task_budgets.py @@ -0,0 +1,71 @@ +"""A run has two concurrency budgets, and a task spends exactly one of them: ``runner`` charges the +agent semaphore (``--max-concurrent``), ``cpu_runner`` the CPU semaphore (``--max-cpu-tasks``).""" + +import asyncio +import enum +from typing import Any, cast + +import pytest + +from composer.io.multi_job import TaskInfo +from composer.pipeline.ptypes import TaskRunnerHost +from composer.rustapp.frontend import GenericRustConsoleHandler + +pytestmark = pytest.mark.asyncio + + +class _Phase(enum.Enum): + WORK = "work" + + +def _host(*, agents: int, cpus: int) -> TaskRunnerHost[_Phase, Any, Any, Any]: + return TaskRunnerHost( + ctx=cast(Any, None), + source=cast(Any, None), + _handler_factory=GenericRustConsoleHandler(set()).make_handler, + _agent_semaphore=asyncio.Semaphore(agents), + _cpu_semaphore=asyncio.Semaphore(cpus), + ) + + +class _Peak: + """Counts how many jobs were ever in flight at once.""" + + def __init__(self) -> None: + self.live = 0 + self.peak = 0 + + async def job(self) -> None: + self.live += 1 + self.peak = max(self.peak, self.live) + # Two hops, so a waiting task gets every chance to slip in if nothing is holding it back. + await asyncio.sleep(0) + await asyncio.sleep(0) + self.live -= 1 + + +def _info(n: int) -> TaskInfo[_Phase]: + return TaskInfo(task_id=f"task-{n}", label=f"Task {n}", phase=_Phase.WORK) + + +async def test_cpu_tasks_are_bounded_by_the_cpu_budget(): + host, peak = _host(agents=8, cpus=2), _Peak() + + await asyncio.gather(*(host.cpu_runner(_info(n), peak.job) for n in range(6))) + + assert peak.peak == 2 + + +async def test_a_cpu_task_does_not_spend_an_agent_slot(): + # The point of the split: a build saturating the CPU budget leaves the agents' concurrency + # exactly as wide as the user asked for. + host = _host(agents=2, cpus=1) + agents, builds = _Peak(), _Peak() + + await asyncio.gather( + *(host.cpu_runner(_info(n), builds.job) for n in range(4)), + *(host.runner(_info(10 + n), agents.job) for n in range(4)), + ) + + assert builds.peak == 1 + assert agents.peak == 2 diff --git a/tests/test_wire_roundtrip.py b/tests/test_wire_roundtrip.py new file mode 100644 index 00000000..6c158aab --- /dev/null +++ b/tests/test_wire_roundtrip.py @@ -0,0 +1,616 @@ +"""Property-based round-trip of the Rust/Python wire protocol, in both directions. + +The two sides of the seam are hand-written mirrors — pydantic models in ``composer.rustapp.wire`` +and ``…descriptor``, serde types in ``rust/autoprover-sdk/src`` — and the docstrings on both ask +that they be kept in lockstep. This is what checks that they were, per field, rather than per +convention. + +Each direction round-trips a payload through the *other* language and back, and asserts nothing was +lost. What a dropped or renamed field looks like: + +* **Outbound** (host → wheel): the host builds a model, dumps it, Rust deserializes into the + mirrored type and re-serializes. A field Rust doesn't declare is dropped on the way through, so + re-parsing the answer no longer equals what was sent. +* **Inbound** (wheel → host): Rust builds a value from entropy Hypothesis drew, serializes it, the + host parses and re-dumps, and Rust serializes the result again. The two Rust-produced documents + are compared, so a field the *host* drops is what shows up as a difference. + +The inbound generator lives in Rust (``autoprover_sdk::fuzz``, behind the ``fuzz`` feature) rather +than being derived from the pydantic schema, and that is the point: a field only the Rust side +declares gets populated. A generator built from the host's own schema can't produce what the host +has never heard of, so it can only ever find drift in one direction. + +A round trip is blind to one thing on a *tolerant* seam: a field only one side declares, which the +other quietly defaults. Both halves ship together, so this seam is not tolerant — nothing on it +defaults anything, and such a field fails at the callout carrying it (see +:mod:`composer.rustapp.wire`). That is what makes the two round trips below sufficient on their own. + +The two field-*set* checks that follow therefore assert nothing the round trips miss; they are kept +because they are deterministic and per-type, so a one-sided field is reported as the field that +diverged rather than as a serde error inside a shrunk example. + +What none of that catches is a payload the harness never hears about: the roots have to be listed, +because a wire name and a direction are not properties of a class. So two checks close that gap by +*discovering* what the ABI defines and requiring the lists to account for it — +``test_every_mirror_is_reachable_from_a_declared_root`` and +``test_every_outbound_mirror_has_its_own_field_set_check``. Add a payload to +:mod:`composer.rustapp.wire` and forget to list it here, and those fail rather than the new payload +being silently exercised by nothing. + +Everything talks to the ``wire-echo`` binary over one long-lived pipe (see its module docs). The +round trips are marked ``wire`` rather than ``fuzz``, since selecting them commits a job to +building that binary; the field-set and completeness checks are deterministic and always run. +""" + +import enum +import functools +import hashlib +import json +import os +import pathlib +import shutil +import subprocess +import sys +import types +import typing +from dataclasses import dataclass +from typing import Annotated, Any, Literal + +import annotated_types +import pytest +from hypothesis import assume, given, settings, strategies as st +from pydantic import BaseModel, TypeAdapter, ValidationError + +#: Imported as modules as well as by name: the completeness checks discover what the ABI defines +#: rather than trusting the import list below to be current. +import composer.rustapp.descriptor as descriptor_abi +import composer.rustapp.wire as wire_abi +from composer.rustapp.descriptor import AppDescriptor +from composer.rustapp.wire import ( + AppArgs, + AuthorInput, + CalloutError, + CompileResult, + ComponentOutcome, + FinalizeComponent, + FinalizeInput, + Judge, + Prompt, + Property, + SkippedProperty, + SandboxGrants, + Target, + Check, + ValidateOutcome, + WireModel, + WorkspacePrep, +) +from composer.sandbox.config import BackendSpec + +REPO_ROOT = pathlib.Path(__file__).parent.parent +RUST_DIR = REPO_ROOT / "rust" +WIRE_ECHO = RUST_DIR / "target" / "debug" / "wire-echo" + + +# --------------------------------------------------------------------------- +# The pipe. +# --------------------------------------------------------------------------- + +class WireFault(AssertionError): + """The Rust side refused a payload — a serde error naming the field that diverged.""" + + +class WireEcho: + """A live ``wire-echo`` process. One per session: an example is a single request/response, and + spawning a process for each would dominate the runtime.""" + + def __init__(self, binary: pathlib.Path) -> None: + self._proc = subprocess.Popen( + [str(binary)], + stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, + text=True, encoding="utf-8", + ) + + def close(self) -> None: + assert self._proc.stdin is not None + self._proc.stdin.close() + self._proc.wait(timeout=10) + + def _request(self, request: dict[str, Any]) -> dict[str, Any]: + assert self._proc.stdin is not None and self._proc.stdout is not None + # ensure_ascii keeps the request line pure ASCII, so drawn text can't depend on the pipe's + # encoding; the protocol is one line per message in both directions. + self._proc.stdin.write(json.dumps(request, ensure_ascii=True) + "\n") + self._proc.stdin.flush() + line = self._proc.stdout.readline() + if not line: + stderr = self._proc.stderr.read() if self._proc.stderr else "" + raise WireFault(f"wire-echo exited (code {self._proc.returncode}): {stderr}") + return json.loads(line) + + def echo(self, ty: str, payload: object) -> object: + """``payload`` deserialized into the Rust ``ty`` and serialized back.""" + answer = self._request({"op": "echo", "ty": ty, "payload": payload}) + match answer: + case {"status": "ok", "payload": echoed}: + return echoed + case _: + raise WireFault(f"echo {ty}: {answer} for {json.dumps(payload)}") + + def gen(self, ty: str, entropy: bytes) -> object | None: + """A Rust-built ``ty``, serialized — or ``None`` when the draw ran out of entropy, which is + this harness's own limit rather than anything about the protocol.""" + answer = self._request({"op": "gen", "ty": ty, "entropy": list(entropy)}) + match answer: + case {"status": "ok", "payload": generated}: + return generated + case {"status": "exhausted"}: + return None + case _: + raise WireFault(f"gen {ty}: {answer}") + + +@pytest.fixture(scope="session") +def wire_echo(): + if shutil.which("cargo") is None: + pytest.skip("no cargo — the Rust half of the protocol can't be built") + subprocess.run( + ["cargo", "build", "--package", "autoprover-sdk", "--features", "fuzz", + "--bin", "wire-echo"], + cwd=RUST_DIR, check=True, timeout=900, + # The SDK pulls in pyo3, whose build script probes the *first* `python3` on PATH and fails + # on one newer than it supports. Pin it to the interpreter running these tests, which is by + # definition a version this checkout supports. + env={**os.environ, "PYO3_PYTHON": sys.executable}, + ) + echo = WireEcho(WIRE_ECHO) + yield echo + echo.close() + + +# --------------------------------------------------------------------------- +# Outbound strategies — what the host sends. +# +# Built by walking each model's ``model_fields`` rather than naming the fields here, because a +# strategy that names them stops covering the one somebody adds next: an undrawn field keeps its +# default, survives the trip, and the round trip passes without ever having tested it. +# --------------------------------------------------------------------------- + +_MAX_ITEMS = 3 +_TEXT = st.text(max_size=24) +#: Bounded to `i64`: past that `serde_json` reads a number as a float, and the comparison would be +#: measuring float formatting rather than the protocol. +_INT = st.integers(min_value=-(2**63), max_value=2**63 - 1) +#: Finite only — NaN and infinity have no JSON spelling, so neither side can carry one. +_FLOAT = st.floats(allow_nan=False, allow_infinity=False) + +#: An opaque payload (`dict[str, Any]`, `serde_json::Value`): any JSON document, kept shallow since +#: both sides treat it as opaque and depth buys no coverage. +_JSON = st.recursive( + st.none() | st.booleans() | _INT | _TEXT, + lambda inner: (st.lists(inner, max_size=_MAX_ITEMS) + | st.dictionaries(_TEXT, inner, max_size=_MAX_ITEMS)), + max_leaves=6, +) + +_LEAVES: dict[Any, st.SearchStrategy[Any]] = { + str: _TEXT, int: _INT, bool: st.booleans(), float: _FLOAT, + type(None): st.none(), Any: _JSON, +} + + +def _bounded(annotation: Any, metadata: tuple[Any, ...]) -> st.SearchStrategy[Any]: + """``annotation`` narrowed by whatever ``annotated_types`` bounds accompany it. + + Honoured rather than ignored because a bound is how this side says what the *protocol's* domain + is, not merely what Python can hold — ``BackendSpec.timeout_s`` is ``Ge(0)`` because the mirrored + field is a ``u64``. Drawing outside it would report a value neither side ever sends.""" + lo = next((m.ge for m in metadata if isinstance(m, annotated_types.Ge)), None) + hi = next((m.le for m in metadata if isinstance(m, annotated_types.Le)), None) + if annotation is int and (lo is not None or hi is not None): + return st.integers(min_value=lo if lo is not None else -(2**63), + max_value=hi if hi is not None else 2**63 - 1) + return _strategy_for(annotation) + + +def _strategy_for(annotation: Any) -> st.SearchStrategy[Any]: + """A strategy for one model field's annotation. + + An annotation this doesn't recognize raises rather than falling back to ``st.from_type``: the + fallback would draw *something* for a field whose wire domain nobody had thought about, which + is how a round trip ends up asserting less than it appears to.""" + if isinstance(annotation, typing.TypeAliasType): # PEP 695 `type X = ...` + return _strategy_for(annotation.__value__) + if annotation in _LEAVES: + return _LEAVES[annotation] + if isinstance(annotation, type) and issubclass(annotation, enum.Enum): + return st.sampled_from(annotation) + if isinstance(annotation, type) and issubclass(annotation, BaseModel): + return _model_strategy(annotation) + if typing.is_typeddict(annotation): + return _typed_dict_strategy(annotation) + origin, args = typing.get_origin(annotation), typing.get_args(annotation) + if origin is Annotated: + return _bounded(args[0], args[1:]) + if origin is Literal: + return st.sampled_from(args) + if origin in (types.UnionType, typing.Union): + return st.one_of(*(_strategy_for(a) for a in args)) + if origin is list: + return st.lists(_strategy_for(args[0]), max_size=_MAX_ITEMS) + if origin is dict: + return st.dictionaries(_strategy_for(args[0]), _strategy_for(args[1]), max_size=_MAX_ITEMS) + if origin is tuple: + return st.tuples(*(_strategy_for(a) for a in args)) + raise NotImplementedError(f"no wire strategy for {annotation!r}; teach _strategy_for about it") + + +def _model_strategy[T: BaseModel](model: type[T]) -> st.SearchStrategy[T]: + """Every field of ``model``, drawn — including ones added after this test was written. + + Constraints come off ``FieldInfo.metadata``, since pydantic strips the ``Annotated`` wrapper + from ``.annotation`` and files the bounds separately.""" + return st.builds(model, **{ + name: _bounded(f.annotation, tuple(f.metadata)) + for name, f in model.model_fields.items() + }) + + +def _typed_dict_strategy(td: Any) -> st.SearchStrategy[dict[str, Any]]: + """Every key of a ``TypedDict`` mirror (``BackendSpec``), drawn. ``include_extras`` keeps the + ``Annotated`` bounds that :func:`_bounded` reads — unlike a model, nothing has stripped them.""" + hints = typing.get_type_hints(td, include_extras=True) + return st.fixed_dictionaries({name: _strategy_for(ann) for name, ann in hints.items()}) + + +def _mirrors_of(payload: Any) -> tuple[Any, ...]: + """The mirrors whose fields land at ``payload``'s **top level**. + + Several when the root is a tagged union, because the tag and the selected variant's fields arrive + together at one level. Unwraps only the containers that add no level of their own — an + ``Annotated`` discriminated union, and a list (a ``Vec`` root presents one ``T``).""" + if typing.is_typeddict(payload) or ( + isinstance(payload, type) and issubclass(payload, BaseModel) + ): + return (payload,) + origin, args = typing.get_origin(payload), typing.get_args(payload) + if origin is Annotated: + return _mirrors_of(args[0]) + if origin in (types.UnionType, typing.Union): + return tuple(m for a in args for m in _mirrors_of(a)) + if origin is list: + return _mirrors_of(args[0]) + raise NotImplementedError(f"no mirrors for payload root {payload!r}") + + +#: Cached because a root's adapter and strategy are asked for once per Hypothesis example, and +#: rebuilding a validator (or a recursive strategy) thousands of times dominates the runtime. +@functools.cache +def _adapter_for(payload: Any) -> TypeAdapter[Any]: + return TypeAdapter(payload) + + +@functools.cache +def _payload_strategy(payload: Any) -> st.SearchStrategy[Any]: + return _strategy_for(payload) + + +@dataclass(frozen=True) +class Root: + """One payload root the seam carries whole: its wire name, and the host annotation describing it. + + Only those two are declared. The adapter, the strategy and the field set are all derived from + ``payload``, so a root cannot describe itself two ways — and adding one means naming it twice + (here and in the Rust ``WireType``), not five times. + + Reading and writing both go through :attr:`adapter` rather than a model's own + ``model_dump_json``, so a mirror that is a ``TypedDict`` (:class:`BackendSpec`) needs no separate + path.""" + + ty: str + payload: Any + + @property + def adapter(self) -> TypeAdapter[Any]: + return _adapter_for(self.payload) + + @property + def strategy(self) -> st.SearchStrategy[Any]: + return _payload_strategy(self.payload) + + @property + def declared(self) -> set[str]: + """The field names at this root's top level.""" + return {name for m in _mirrors_of(self.payload) for name in _field_names(m)} + + +#: Host → wheel. `author_input` is the one root the host models as a union of variants rather than +#: one struct: the Rust mirror is a single struct with the payload flattened in, and the tag is all +#: that says which fields should be there. `sandbox` is the one mirrored by a `TypedDict`. +OUTBOUND = [ + Root("app_args", AppArgs), + Root("author_input", AuthorInput), + Root("target", Target), + Root("finalize_input", FinalizeInput), + Root("sandbox", BackendSpec), +] + +#: Only ever nested inside the outbound :class:`Target` now that the check *names* are the author's: +#: no callout returns checks, so this crosses the seam host → wheel and never back. +_CHECKS = Root("checks", list[Check]) + +#: Wheel → host. +INBOUND = [ + Root("app_descriptor", AppDescriptor), + Root("compile_result", CompileResult), + Root("validate_outcome", ValidateOutcome), + Root("prompt", Prompt), + Root("judge", Judge), + Root("workspace_prep", WorkspacePrep), + Root("sandbox_grants", SandboxGrants), + Root("callout_error", CalloutError), +] + + +# --------------------------------------------------------------------------- +# The round trips. +# --------------------------------------------------------------------------- + +#: Examples per payload root, pinned rather than left to the active Hypothesis profile. What this +#: test asserts is a contract, so it should assert the same amount on every run — and the profile in +#: effect during a full suite run is whichever one another module loaded at import time, which is no +#: way to decide how hard a protocol gets checked. An example is one pipe round trip, so this is +#: a few seconds for the whole file. +_EXAMPLES = 200 + +# `deadline=None` throughout: an example spans a subprocess round trip, so per-example timing is +# noisy enough to trip the default deadline for reasons unrelated to the protocol. + +@pytest.mark.wire +@pytest.mark.parametrize("case", OUTBOUND, ids=[c.ty for c in OUTBOUND]) +@settings(deadline=None, max_examples=_EXAMPLES) +@given(data=st.data()) +def test_outbound_payload_survives_the_wheel( + case: Root, wire_echo: WireEcho, data: st.DataObject +) -> None: + """A payload the host sends reaches the wheel whole: every field the host set is one the Rust + type declares, so re-reading what Rust made of it gives the value back unchanged.""" + sent = data.draw(case.strategy) + echoed = wire_echo.echo(case.ty, json.loads(case.adapter.dump_json(sent))) + assert case.adapter.validate_python(echoed) == case.adapter.validate_python(sent) + + +@pytest.mark.wire +@pytest.mark.parametrize("case", INBOUND, ids=[c.ty for c in INBOUND]) +@settings(deadline=None, max_examples=_EXAMPLES) +@given(entropy=st.binary(min_size=1, max_size=512)) +def test_inbound_payload_survives_the_host(case: Root, wire_echo: WireEcho, entropy: bytes) -> None: + """A payload a wheel returns reaches the host whole. Both documents compared here are Rust's + own serialization, which is what lets the comparison ignore how the two languages spell an + absent optional (see ``test_an_empty_optional_is_spelled_null_on_both_sides``) and see only content.""" + produced = wire_echo.gen(case.ty, entropy) + assume(produced is not None) + reparsed = case.adapter.validate_python(produced) + redumped = json.loads(case.adapter.dump_json(reparsed)) + assert wire_echo.echo(case.ty, redumped) == produced + + +#: Draws for the coverage check below. Deterministic (not a Hypothesis property) so the union of +#: fields it observes is the same on every run: a coverage assertion that sometimes passes would be +#: worse than none. +_COVERAGE_DRAWS = 250 + + +def _coverage_entropy(draw: int) -> bytes: + """Varied but fixed bytes for draw number ``draw``.""" + return hashlib.sha256(str(draw).encode()).digest() * 8 + + +def _field_names(mirror: Any) -> set[str]: + """The field names of one host-side mirror, model or ``TypedDict``.""" + if typing.is_typeddict(mirror): + return set(typing.get_type_hints(mirror)) + return set(mirror.model_fields) + + +def _keys_anywhere(document: object) -> set[str]: + """Every object key in ``document``, at any depth.""" + match document: + case dict(): + return set(document).union(*(_keys_anywhere(v) for v in document.values())) + case list(): + return set().union(*(_keys_anywhere(v) for v in document)) + case _: + return set() + + +def _declared_anywhere(adapter: TypeAdapter[Any]) -> set[str]: + """Every field name in the host's model tree for a payload root, read off its JSON schema so + this doesn't need a second annotation walker to keep in step with ``_strategy_for``.""" + def walk(node: object) -> set[str]: + match node: + case {"properties": dict() as props}: + return set(props).union(*(walk(v) for v in node.values())) + case dict(): + return set().union(*(walk(v) for v in node.values())) + case list(): + return set().union(*(walk(v) for v in node)) + case _: + return set() + + return walk(adapter.json_schema()) + + +@pytest.mark.parametrize("case", INBOUND, ids=[c.ty for c in INBOUND]) +def test_generator_reaches_every_field_the_host_declares(case: Root, wire_echo: WireEcho) -> None: + """Every field the host declares on an inbound payload is one a wheel actually sends. + + This is the round trip's blind spot, and the reason it needs covering separately: a field only + the *host* declares, with a default, is exactly what forward compatibility is supposed to + tolerate — an older wheel omitting it leaves it at its default, and the round trip cannot tell + that from a field no wheel will ever send, i.e. a typo or a leftover. What Rust's generator + never emits, nothing emits.""" + observed: set[str] = set() + for draw in range(_COVERAGE_DRAWS): + produced = wire_echo.gen(case.ty, _coverage_entropy(draw)) + if produced is not None: + observed |= _keys_anywhere(produced) + missing = _declared_anywhere(case.adapter) - observed + assert not missing, f"{case.ty}: no wheel can send {sorted(missing)} — absent from the Rust type" + + +#: Structs that only ever travel *inside* an outbound payload. Addressable in their own right so the +#: field-set check can generate one alone: an opaque payload's keys sit at the same depths as a nested +#: struct's, and are not field names at all, so comparing each struct's own top level is what keeps +#: that assertion exact. +NESTED = [ + _CHECKS, + Root("property", Property), + Root("skipped_property", SkippedProperty), + Root("finalize_component", FinalizeComponent), + Root("component_outcome", ComponentOutcome), +] + +#: Everything reachable on an outbound payload — the roots, plus the structs nested in them. Derived +#: rather than listed again, so the two can't disagree about which roots exist. +MIRRORS = [*OUTBOUND, *NESTED] + + +#: The modules that between them define every mirror on the seam. Discovery reads these rather than a +#: hand-kept list, so the completeness check below has an independent source of truth. +_ABI_MODULES = (wire_abi, descriptor_abi) + + +def _defined_mirrors() -> set[Any]: + """Every mirror the ABI modules define, plus the sandbox layer's one. + + Filtered by ``__module__`` so a name one module imports from another is counted once, where it is + defined; underscore-prefixed models (``_AuthorInputBase``) are internal shape-sharing, never a + payload of their own.""" + return {BackendSpec} | { + obj + for module in _ABI_MODULES + for name, obj in vars(module).items() + if not name.startswith("_") + and isinstance(obj, type) + and issubclass(obj, WireModel) + and obj is not WireModel + and obj.__module__ == module.__name__ + } + + +def _mirrors_within(annotation: Any, found: set[Any]) -> None: + """Collect into ``found`` every mirror ``annotation`` can reach, at any depth.""" + if isinstance(annotation, typing.TypeAliasType): + _mirrors_within(annotation.__value__, found) + return + if typing.is_typeddict(annotation): + if annotation in found: + return + found.add(annotation) + for field in typing.get_type_hints(annotation, include_extras=True).values(): + _mirrors_within(field, found) + return + if isinstance(annotation, type) and issubclass(annotation, BaseModel): + if annotation in found: + return + found.add(annotation) + for field in annotation.model_fields.values(): + _mirrors_within(field.annotation, found) + return + for arg in typing.get_args(annotation): + _mirrors_within(arg, found) + + +def test_every_outbound_mirror_has_its_own_field_set_check() -> None: + """Every struct reachable on an outbound payload is compared in its *own* right. + + :data:`MIRRORS` has to name each nested struct, because the field-set check compares only a + struct's own top level — reach `Target` and you see ``{name, units}``, never a `Check`'s fields. + The wire names are the Rust type names, which nothing here can derive without munging a string, + so the list stays declared and this is the half that keeps it complete.""" + reached: set[Any] = set() + for root in OUTBOUND: + _mirrors_within(root.payload, reached) + checked = {mirror for root in MIRRORS for mirror in _mirrors_of(root.payload)} + assert not reached - checked, ( + f"reachable on an outbound payload but never compared on its own: " + f"{sorted(m.__name__ for m in reached - checked)} — add it to NESTED" + ) + + +def test_every_mirror_is_reachable_from_a_declared_root() -> None: + """Every mirror the ABI defines is carried by one of the roots above. + + The roots themselves stay declared: a wire name and a direction are not properties of a class, so + nothing can infer them. What this closes is the gap that leaves — add a payload to + ``composer.rustapp.wire`` and forget to list it, and it is silently exercised by nothing. Here it + is discovery against declaration, so the omission fails instead.""" + reached: set[Any] = set() + for root in [*OUTBOUND, *INBOUND, *NESTED]: + _mirrors_within(root.payload, reached) + missed = _defined_mirrors() - reached + assert not missed, ( + f"no declared root reaches {sorted(m.__name__ for m in missed)} — add it to OUTBOUND or " + "INBOUND if it is a payload root of its own, or to the payload that carries it if not" + ) + + +def _top_level_keys(document: object) -> set[str]: + """The keys of ``document`` itself — not of anything nested inside it. A list stands for its + elements, which is how a ``Vec`` root presents one ``T``.""" + match document: + case dict(): + return set(document) + case list(): + return set().union(*(_top_level_keys(item) for item in document)) + case _: + return set() + + +@pytest.mark.parametrize("case", MIRRORS, ids=[m.ty for m in MIRRORS]) +def test_wheel_reads_no_field_the_host_never_sends(case: Root, wire_echo: WireEcho) -> None: + """Every field a wheel can read off an outbound payload is one the host actually sends. + + The mirror of ``test_generator_reaches_every_field_the_host_declares``, and the round trip's + other blind spot: a field only the *Rust* side declares is filled by ``#[serde(default)]`` — or, + for an ``Option``, by serde regardless — so the payload deserializes, the field is dropped + again on the way back out, and the round trip sees nothing wrong. What the wheel would actually + read is an empty string, a ``None`` or an empty vec, forever, for something no host ever sets.""" + emitted: set[str] = set() + for draw in range(_COVERAGE_DRAWS): + produced = wire_echo.gen(case.ty, _coverage_entropy(draw)) + if produced is not None: + emitted |= _top_level_keys(produced) + unknown = emitted - case.declared + assert not unknown, f"{case.ty}: a wheel can read {sorted(unknown)}, which no host sends" + + +def test_an_empty_optional_is_spelled_null_on_both_sides(wire_echo: WireEcho) -> None: + """An empty optional has exactly one spelling, and it is ``null``. + + Nothing on this seam omits a key: Rust carries no ``skip_serializing_if`` and the inbound models + default nothing, so absence is an error on whichever side reads it rather than a second way to + say "nothing". That is what lets the round trips above compare documents directly — they are + comparing content, not two conventions for the same value.""" + row = {"name": "u", "properties": ["p"], "target": None} + assert wire_echo.echo("checks", [row]) == [row] + assert Check.model_validate(row) == Check(name="u", properties=["p"], target=None) + + absent = {"name": "u", "properties": ["p"]} + with pytest.raises(WireFault, match="missing field"): + wire_echo.echo("checks", [absent]) + with pytest.raises(ValidationError): + Check.model_validate(absent) + + +def test_descriptor_rejects_unknown_tags(wire_echo: WireEcho) -> None: + """``ecosystem`` and ``backend_tag`` are plain ``String`` in Rust but closed sets here, so the + inbound round trip draws from those sets (``autoprover_sdk::fuzz``) instead of reporting every + unknown tag as drift. That narrowing is only sound if the host does reject the rest.""" + produced = wire_echo.gen("app_descriptor", bytes(range(64))) + assert isinstance(produced, dict) + AppDescriptor.model_validate(produced) + for field in ("ecosystem", "backend_tag"): + with pytest.raises(ValidationError): + AppDescriptor.model_validate({**produced, field: "nonesuch"}) diff --git a/uv.lock b/uv.lock index a769dcfb..a5afff36 100644 --- a/uv.lock +++ b/uv.lock @@ -127,9 +127,20 @@ s3 = [ ] [package.dev-dependencies] +apps = [ + { name = "echoprover" }, + { name = "maturin-import-hook" }, + { name = "run-confined", marker = "sys_platform == 'linux' or (extra == 'extra-11-ai-composer-certora-cli' and extra == 'extra-11-ai-composer-certora-cli-beta') or (extra == 'extra-11-ai-composer-certora-cli' and extra == 'extra-11-ai-composer-certora-cli-beta-mirror') or (extra == 'extra-11-ai-composer-certora-cli-beta' and extra == 'extra-11-ai-composer-certora-cli-beta-mirror') or (extra == 'extra-11-ai-composer-cpu' and extra == 'extra-11-ai-composer-cuda')" }, +] ci = [ { name = "pyright" }, ] +dev = [ + { name = "echoprover" }, + { name = "maturin" }, + { name = "maturin-import-hook" }, + { name = "run-confined", marker = "sys_platform == 'linux' or (extra == 'extra-11-ai-composer-certora-cli' and extra == 'extra-11-ai-composer-certora-cli-beta') or (extra == 'extra-11-ai-composer-certora-cli' and extra == 'extra-11-ai-composer-certora-cli-beta-mirror') or (extra == 'extra-11-ai-composer-certora-cli-beta' and extra == 'extra-11-ai-composer-certora-cli-beta-mirror') or (extra == 'extra-11-ai-composer-cpu' and extra == 'extra-11-ai-composer-cuda')" }, +] ragbuild = [ { name = "beautifulsoup4" }, { name = "einops" }, @@ -153,7 +164,7 @@ requires-dist = [ { name = "aiohttp", specifier = ">=3.13" }, { name = "anthropic", specifier = ">=0.18.0" }, { name = "attrs", specifier = ">=26.1" }, - { name = "certora-cli", marker = "extra == 'certora-cli'", specifier = ">=8.18.0" }, + { name = "certora-cli", marker = "extra == 'certora-cli'", specifier = ">=8.19.1" }, { name = "certora-cli-beta", marker = "extra == 'certora-cli-beta'", specifier = ">=8.18.0" }, { name = "certora-cli-beta-mirror", marker = "extra == 'certora-cli-beta-mirror'", specifier = ">=8.18.0" }, { name = "certora-prover-cli" }, @@ -194,7 +205,18 @@ requires-dist = [ provides-extras = ["ml", "s3", "certora-cli", "certora-cli-beta", "certora-cli-beta-mirror", "prover", "cpu", "cuda"] [package.metadata.requires-dev] +apps = [ + { name = "echoprover", editable = "rust/example-app" }, + { name = "maturin-import-hook", specifier = ">=0.3.0" }, + { name = "run-confined", marker = "sys_platform == 'linux'", directory = "rust/run-confined" }, +] ci = [{ name = "pyright" }] +dev = [ + { name = "echoprover", editable = "rust/example-app" }, + { name = "maturin", specifier = ">=1.14.1" }, + { name = "maturin-import-hook", specifier = ">=0.3.0" }, + { name = "run-confined", marker = "sys_platform == 'linux'", directory = "rust/run-confined" }, +] ragbuild = [ { name = "beautifulsoup4", specifier = "==4.13.5" }, { name = "einops" }, @@ -629,7 +651,7 @@ wheels = [ [[package]] name = "certora-cli" -version = "8.18.0" +version = "8.19.1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "click" }, @@ -645,11 +667,11 @@ dependencies = [ { name = "typing-extensions" }, { name = "wcmatch" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/63/2b/d7c4f7f8c6509def6289da1fdd9f41e764269160180e7b1ad8bcbb596ced/certora_cli-8.18.0.tar.gz", hash = "sha256:d059116e852aafbfbfca7f6d8090968e9ea2bbcc056989ed9a282e3f6fed07dd", size = 43347710, upload-time = "2026-07-30T09:09:51.057Z" } +sdist = { url = "https://files.pythonhosted.org/packages/ad/cd/0bcd2b5ff1b5f2b31ef18e59e6316d194b88cde0e8e7161fc88b345dd84f/certora_cli-8.19.1.tar.gz", hash = "sha256:05a429ebd2463df1b6c5aae6ed65b3585dd13bee668f2160193e797e38d5f4d9", size = 43395773, upload-time = "2026-08-28T20:01:40.072Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/44/47/a7fbf0247b7fc7b0319709ec0b58814503da1c7d2e09129b5103b968a1e1/certora_cli-8.18.0-py3-none-any.whl", hash = "sha256:a13d46298784561a5c71cfdf103d3578df944a8514a859751a609c6ac175dc31", size = 43402889, upload-time = "2026-07-30T09:09:43.12Z" }, - { url = "https://files.pythonhosted.org/packages/de/c9/b38f13ec5807e8fb03a3ad30737b0d0997d6d62c17abed058043fce767d3/certora_cli-8.18.0-py3-none-macosx_10_9_universal2.whl", hash = "sha256:441269bc7bd2a8d77955660403c361ac8b6ec051a3b82618c706e55693231c5d", size = 44929014, upload-time = "2026-07-30T09:09:48.593Z" }, - { url = "https://files.pythonhosted.org/packages/86/76/d957d151848e091c4bec1014d15e6658d2c1a06463f195236085e1b17bf6/certora_cli-8.18.0-py3-none-manylinux_2_28_x86_64.whl", hash = "sha256:7d0849908bd8b8102424e8e63fd91afae0aee174cd750d4f874792e3614876bf", size = 44238961, upload-time = "2026-07-30T09:09:45.825Z" }, + { url = "https://files.pythonhosted.org/packages/f7/05/4e919576e253c16894e842d08db1e03b62ecfb1b785ea0738c1dea16b73e/certora_cli-8.19.1-py3-none-any.whl", hash = "sha256:2e4b1c9c79fe84bb91eb116f241d8a58923859f2d198d7a7fb0af4843bfcb2ab", size = 43450543, upload-time = "2026-08-28T20:01:31.239Z" }, + { url = "https://files.pythonhosted.org/packages/47/43/da1321852dbb753d871d07bb152a062b30123f7e276616eab96a11c63d89/certora_cli-8.19.1-py3-none-macosx_10_9_universal2.whl", hash = "sha256:689f59dce81d9b97298b349e50f864740f62740478180c7dc61dd2ed3f02d738", size = 44976667, upload-time = "2026-08-28T20:01:36.925Z" }, + { url = "https://files.pythonhosted.org/packages/39/f0/b0f2b24be05a7cfc8f6d302ff78d5812e346ab0d374b04aba0776a73630d/certora_cli-8.19.1-py3-none-manylinux_2_28_x86_64.whl", hash = "sha256:9ae2c66b0399c4a472f80f62e5f7600ed7528b5deedaa18f6dc257db9ad077a9", size = 44286614, upload-time = "2026-08-28T20:01:33.972Z" }, ] [[package]] @@ -725,7 +747,7 @@ wheels = [ [[package]] name = "certora-prover-cli" -version = "0.1.0" +version = "0.2.1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "boto3" }, @@ -734,9 +756,9 @@ dependencies = [ { name = "requests" }, { name = "urllib3" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/e7/2b/74f10094265c2bb3ed9c246266534b7f01363d7ec5ff19038932086823f4/certora_prover_cli-0.1.0.tar.gz", hash = "sha256:954aba1d9ec5db8e3f15e0e38768de7609c0f0e11a6a7efed7441c756c72d981", size = 65865, upload-time = "2026-06-18T19:33:25.77Z" } +sdist = { url = "https://files.pythonhosted.org/packages/d6/54/73dc7f810bd4c8b8da92ca1f1074d4dd6d8edbb537cde6f13a9f7c845c34/certora_prover_cli-0.2.1.tar.gz", hash = "sha256:ee6f4d95a440e7be3edfed6d93f7ba07386482e5eb939a41128a48f1198a33de", size = 67439, upload-time = "2026-08-28T10:51:36.83Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/37/e7/1b6c307f7ddebb2d3b237966925dc60d5161c3ce6b2848939b82884e62d5/certora_prover_cli-0.1.0-py3-none-any.whl", hash = "sha256:ec7c6349cc3ede32f46c6d1cf63e73864dbfcd2d5b643aa526b685b861ccc205", size = 78393, upload-time = "2026-06-18T19:33:24.393Z" }, + { url = "https://files.pythonhosted.org/packages/c5/d9/fca95f34a9598bdd5c0df9bf8ff5787137786bbb49cca873d6c34bac6bac/certora_prover_cli-0.2.1-py3-none-any.whl", hash = "sha256:72dfdf20d204403c3bb3c353b76d02171fb27a91082b4a41f82d4878f6061782", size = 79828, upload-time = "2026-08-28T10:51:35.455Z" }, ] [[package]] @@ -1269,6 +1291,11 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/b0/0d/9feae160378a3553fa9a339b0e9c1a048e147a4127210e286ef18b730f03/durationpy-0.10-py3-none-any.whl", hash = "sha256:3b41e1b601234296b4fb368338fdcd3e13e0b4fb5b67345948f4f2bf9868b286", size = 3922, upload-time = "2025-05-17T13:52:36.463Z" }, ] +[[package]] +name = "echoprover" +version = "0.1.0" +source = { editable = "rust/example-app" } + [[package]] name = "einops" version = "0.8.2" @@ -1447,6 +1474,7 @@ requires-dist = [ [package.metadata.requires-dev] ci = [{ name = "pyright" }] test = [ + { name = "hypothesis" }, { name = "pytest", specifier = ">=9.0" }, { name = "pytest-asyncio", specifier = ">=1.3" }, { name = "testcontainers", specifier = ">=4.0" }, @@ -2394,6 +2422,39 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/70/bc/6f1c2f612465f5fa89b95bead1f44dcb607670fd42891d8fdcd5d039f4f4/markupsafe-3.0.3-cp314-cp314t-win_arm64.whl", hash = "sha256:32001d6a8fc98c8cb5c947787c5d08b0a50663d139f1305bac5885d98d9b40fa", size = 14146, upload-time = "2025-09-27T18:37:28.327Z" }, ] +[[package]] +name = "maturin" +version = "1.14.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/e7/b3/addd877f871fb1860d46d3a4f206ecb10b946c85846805e6367631926fd3/maturin-1.14.1.tar.gz", hash = "sha256:9d6577a62cd08e0ceba7a0db06fb098e0c9b1b3429bad747a4f3a18215a1b3df", size = 369637, upload-time = "2026-06-19T05:19:49.774Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f4/f0/97c5a5bd9c71653a066c0976a484eaaae50b9369557838a4176b7b0bdaa5/maturin-1.14.1-py3-none-linux_armv6l.whl", hash = "sha256:522292398945442cdafa9daeb2271b2340fbde57027b818f923f88eab04174f8", size = 10207496, upload-time = "2026-06-19T05:19:09.321Z" }, + { url = "https://files.pythonhosted.org/packages/fe/83/294bca639b0e052f1e2f65199b3db258780c7d4e31408b934c9c974a1379/maturin-1.14.1-py3-none-macosx_10_12_x86_64.macosx_11_0_arm64.macosx_10_12_universal2.whl", hash = "sha256:ffe5ad71f21d1e6603c4dd75f7fee34adf5ed5ebcebb692886549888ebb329ed", size = 19680113, upload-time = "2026-06-19T05:19:13.43Z" }, + { url = "https://files.pythonhosted.org/packages/43/b6/79c881410a3b1c187f7eb3d407aecae646c6a4433d630d72200359015e83/maturin-1.14.1-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:f3306078070c1508fd715b9116070cbcaff5959024272a9f1e6f5cb29768b86c", size = 10169205, upload-time = "2026-06-19T05:19:16.615Z" }, + { url = "https://files.pythonhosted.org/packages/93/9d/44b6f26dcb7f7a04c5501ac2dbb6ca1490150682baa525ca5860504f9eab/maturin-1.14.1-py3-none-manylinux_2_12_i686.manylinux2010_i686.musllinux_1_1_i686.whl", hash = "sha256:cd457cd88961156e26379e1155bd287cc0ec1c8b2f1582b0660fb31b87c8842d", size = 10188098, upload-time = "2026-06-19T05:19:19.736Z" }, + { url = "https://files.pythonhosted.org/packages/1a/bd/9c0d5d6983905ce2c9edaa073a7e89355a9cf7f396988e05d32f1c37785d/maturin-1.14.1-py3-none-manylinux_2_12_x86_64.manylinux2010_x86_64.musllinux_1_1_x86_64.whl", hash = "sha256:dfc54ae32e6fcb18302193ab9a30b0b25eefffba994ae13238974805533ef75e", size = 10627576, upload-time = "2026-06-19T05:19:22.713Z" }, + { url = "https://files.pythonhosted.org/packages/e5/33/b096412bd6a7cb399652b260666f901adf88a687181a6dbd6a3f89f0a94e/maturin-1.14.1-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.musllinux_1_1_aarch64.whl", hash = "sha256:a131d912b5267e640bc96d70f4914e10590aed64082ec9abacba7cea52004224", size = 10085181, upload-time = "2026-06-19T05:19:25.69Z" }, + { url = "https://files.pythonhosted.org/packages/56/8d/08c3bf469c38a23c9e6c877e338193001eb604d010fedc08341974e38528/maturin-1.14.1-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.musllinux_1_1_armv7l.whl", hash = "sha256:be18fc568fb76884c0205456336892a75105ec398e6b667cd777c6268bd06d69", size = 10026363, upload-time = "2026-06-19T05:19:28.904Z" }, + { url = "https://files.pythonhosted.org/packages/3a/a4/c4d1a92839f8745ab4aab988a7db884a79d6d710bd3b286fcf9316dece1a/maturin-1.14.1-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.musllinux_1_1_ppc64le.whl", hash = "sha256:994a0c8ba3ad8a92b3a9ee1b02645d200d610216b15cff5102b0fe65e8e08666", size = 13321347, upload-time = "2026-06-19T05:19:32.411Z" }, + { url = "https://files.pythonhosted.org/packages/b3/fa/170f04624d03fd07d2a8b1b67de83a127af93aef9eaa425839553347297b/maturin-1.14.1-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:be80866363e605d137991b491a741a84cde9ae350183c4c85f49690ca9aaaa65", size = 10877609, upload-time = "2026-06-19T05:19:35.448Z" }, + { url = "https://files.pythonhosted.org/packages/61/ad/1ae2e1d0ded282bf2c55ac13f0811d87deb425e200ae64a15785675dede9/maturin-1.14.1-py3-none-manylinux_2_31_riscv64.musllinux_1_1_riscv64.whl", hash = "sha256:5282dffd4b539d2be245f4e5b1a5ab6bc1033b58f4a4872f5833f9d43c954aa4", size = 10417316, upload-time = "2026-06-19T05:19:38.28Z" }, + { url = "https://files.pythonhosted.org/packages/fb/27/bf677183920718da49cd7982d6a3ffc440aad8919329f571d189f81b7bdf/maturin-1.14.1-py3-none-win32.whl", hash = "sha256:1a04de0a20188f95c721b5702eed18140bdcccb28c386797093eca3f62f4d4e0", size = 8931293, upload-time = "2026-06-19T05:19:41.183Z" }, + { url = "https://files.pythonhosted.org/packages/63/4b/585adeb9167b08d3cdff0032a938b0e72655c92003df4f52c3f696a1bcc2/maturin-1.14.1-py3-none-win_amd64.whl", hash = "sha256:3c9f94640ecc4895e94abaf834a0684430032c865b2748a36c12461fd9252fdd", size = 10314067, upload-time = "2026-06-19T05:19:44.389Z" }, + { url = "https://files.pythonhosted.org/packages/51/d4/dac8c0720ae246be1700afb6fbdbbea20fe35b13f6570b2f70faa005df77/maturin-1.14.1-py3-none-win_arm64.whl", hash = "sha256:15cea8fcb3ba47dd636f50092bb34baea8b04ac777392f23e6bf8a9a61efb894", size = 9718943, upload-time = "2026-06-19T05:19:47.49Z" }, +] + +[[package]] +name = "maturin-import-hook" +version = "0.3.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "filelock" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/75/a1/3b007a4132265d5da19096d6835cd4c7e9908afb9156afb96d19ff190f39/maturin_import_hook-0.3.0.tar.gz", hash = "sha256:cdd51ce267663c437c4aea1bfd5f7ee5884b54a9d866ea924227d68d758bbba2", size = 30785, upload-time = "2025-06-08T15:43:57.801Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9f/55/0c17a1fccec90dffbb1d904d7ddb7971e4e95f994c6264637e8fe66a5b86/maturin_import_hook-0.3.0-py3-none-any.whl", hash = "sha256:f5d20b4bc3056d84170dc08fe655160421b28d2b5128b48f55312315cdbb2cfe", size = 30848, upload-time = "2025-06-08T15:43:56.298Z" }, +] + [[package]] name = "mcp" version = "1.28.0" @@ -4376,6 +4437,11 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/d0/02/fa464cdfbe6b26e0600b62c528b72d8608f5cc49f96b8d6e38c95d60c676/rpds_py-0.30.0-cp314-cp314t-win_amd64.whl", hash = "sha256:27f4b0e92de5bfbc6f86e43959e6edd1425c33b5e69aab0984a72047f2bcf1e3", size = 226532, upload-time = "2025-11-30T20:24:14.634Z" }, ] +[[package]] +name = "run-confined" +version = "0.1.0" +source = { directory = "rust/run-confined" } + [[package]] name = "s3fs" version = "2026.3.0"