diff --git a/.github/workflows/publish-pypi.yml b/.github/workflows/publish-pypi.yml index e26033738..4e363308d 100644 --- a/.github/workflows/publish-pypi.yml +++ b/.github/workflows/publish-pypi.yml @@ -50,6 +50,9 @@ jobs: uses: "actions/checkout@v7" with: submodules: true + # Full history + tags so hatch-vcs stamps the real tag version at build time, not the fallback + fetch-depth: 0 + fetch-tags: true - name: Cache UV dependencies uses: "actions/cache@v6" diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 44de92c21..09c8363fe 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -24,6 +24,9 @@ jobs: uses: "actions/checkout@v7" with: submodules: true + # Full history + tags so hatch-vcs resolves the exact tag version from installed metadata + fetch-depth: 0 + fetch-tags: true - name: "Set up Python" uses: "actions/setup-python@v7" @@ -41,10 +44,13 @@ jobs: - name: Check prerelease type id: release run: | - VERSION=$(uv version --short) + VERSION=$(uv run python -c "import importlib.metadata; print(importlib.metadata.version('infrahub-sdk'))") + echo "version=$VERSION" >> "$GITHUB_OUTPUT" echo is_prerelease=$(uv run python -c "from packaging.version import Version; print(int(Version('$VERSION').is_prerelease))") >> "$GITHUB_OUTPUT" echo is_devrelease=$(uv run python -c "from packaging.version import Version; print(int(Version('$VERSION').is_devrelease))") >> "$GITHUB_OUTPUT" - echo "version=$VERSION" >> "$GITHUB_OUTPUT" + echo is_local=$(uv run python -c "from packaging.version import Version; print(int(Version('$VERSION').local is not None))") >> "$GITHUB_OUTPUT" + echo base_version=$(uv run python -c "from packaging.version import Version; print(Version('$VERSION').base_version)") >> "$GITHUB_OUTPUT" + echo fallback_base=$(uv run python -c "import tomllib; from packaging.version import Version; print(Version(tomllib.load(open('pyproject.toml', 'rb'))['tool']['hatch']['version']['fallback-version']).base_version)") >> "$GITHUB_OUTPUT" echo major_minor_version=$(uv run python -c "from packaging.version import Version; v = Version('$VERSION'); print(f'{v.major}.{v.minor}')") >> "$GITHUB_OUTPUT" echo latest_tag=$(curl -L \ -H "Accept: application/vnd.github+json" \ @@ -53,16 +59,23 @@ jobs: https://api.github.com/repos/${{ github.repository }}/releases/latest \ | jq -r '.tag_name') >> "$GITHUB_OUTPUT" - - name: Check tag version + - name: "Publish guard: resolved version must match the release tag" run: | EXPECTED_TAG="v${{ steps.release.outputs.version }}" if [ "${{ github.event.release.tag_name }}" != "$EXPECTED_TAG" ]; then - echo "Tag version does not match python project version" + echo "Resolved version (${{ steps.release.outputs.version }}) does not match release tag ${{ github.event.release.tag_name }}" echo "Expected: $EXPECTED_TAG" echo "Got: ${{ github.event.release.tag_name }}" exit 1 fi + - name: "Publish guard: reject unreleased fallback version" + # fallback_base is read from pyproject.toml at run time; the fallback is a static sentinel (0.0.0.dev0) + if: steps.release.outputs.base_version == steps.release.outputs.fallback_base && (steps.release.outputs.is_devrelease == 1 || steps.release.outputs.is_local == 1) + run: | + echo "Resolved version (${{ steps.release.outputs.version }}) is an unreleased fallback (base ${{ steps.release.outputs.fallback_base }}, dev/local build): no v* tag is reachable. Refusing to publish." + exit 1 + - name: Check prerelease and project version if: github.event.release.prerelease == true && steps.release.outputs.is_prerelease == 0 && steps.release.outputs.is_devrelease == 0 run: | diff --git a/.github/workflows/repository-dispatch.yml b/.github/workflows/repository-dispatch.yml index 41437d265..707c12c53 100644 --- a/.github/workflows/repository-dispatch.yml +++ b/.github/workflows/repository-dispatch.yml @@ -37,6 +37,7 @@ jobs: repo: - "opsmill/emma" - "opsmill/infrahub-demo-dc" + - "opsmill/infrahub-sync" - "INFRAHUB_CUSTOMER1_REPOSITORY" steps: diff --git a/.github/workflows/schema-drift.yml b/.github/workflows/schema-drift.yml new file mode 100644 index 000000000..86e37c453 --- /dev/null +++ b/.github/workflows/schema-drift.yml @@ -0,0 +1,32 @@ +--- +# yamllint disable rule:truthy rule:line-length +name: Schema Drift Check + +# Warn-only: surfaces when the published Infrahub JSON schema has drifted from +# the formatter's committed baseline (infrahub_sdk/ctl/schema_properties.json). +# This never fails the run — it emits ::warning:: annotations and a job summary +# so a maintainer can account for the change in schema_format.py. + +on: + release: + types: + - published + workflow_dispatch: + +jobs: + schema-drift: + runs-on: "ubuntu-22.04" + timeout-minutes: 5 + steps: + - name: "Check out repository code" + uses: "actions/checkout@v6" + - name: Set up Python + uses: actions/setup-python@v6 + with: + python-version: "3.12" + - name: Install UV + uses: astral-sh/setup-uv@v7 + - name: Install dependencies + run: uv sync --all-groups --all-extras + - name: "Check for Infrahub schema drift (warn only)" + run: uv run invoke schema-drift-check diff --git a/.gitignore b/.gitignore index d4efe98f8..d57e60169 100644 --- a/.gitignore +++ b/.gitignore @@ -30,8 +30,14 @@ dist/* # Generated files generated/ +# Committed, generated schema models (write/read variants) must be version-controlled. +!infrahub_sdk/schema/generated/ +!infrahub_sdk/schema/generated/*.py sandbox/ +# hatch-vcs version file (written at build time; must not be tracked) +infrahub_sdk/_version.py + # SpecKit internal cache .specify/**/.cache/ .specify/feature.json diff --git a/.vale/styles/Infrahub/sentence-case.yml b/.vale/styles/Infrahub/sentence-case.yml index c27cf7a1e..5ea115fc9 100644 --- a/.vale/styles/Infrahub/sentence-case.yml +++ b/.vale/styles/Infrahub/sentence-case.yml @@ -45,6 +45,7 @@ exceptions: - IP - IP Fabric - IPAM + - IPAddress - IPHost - IPNetwork - JavaScript diff --git a/.vale/styles/spelling-exceptions.txt b/.vale/styles/spelling-exceptions.txt index 068b304a0..c0e56824a 100644 --- a/.vale/styles/spelling-exceptions.txt +++ b/.vale/styles/spelling-exceptions.txt @@ -8,6 +8,7 @@ artifact_definitions artifact_name async Authentik +backoff boolean check_definitions class_name diff --git a/CHANGELOG.md b/CHANGELOG.md index d71ddffa9..7e6a7ac06 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,37 @@ This project uses [*towncrier*](https://towncrier.readthedocs.io/) and the chang +## [1.23.0](https://github.com/opsmill/infrahub-sdk-python/tree/v1.23.0) - 2026-08-19 + +### Removed + +- Removed `client.branch.diff_data()` from both the async and sync clients. The method relied on a `GET /api/diff/data` REST endpoint that does not exist in Infrahub, so every call returned a 404. Use `client.get_diff_tree()` to retrieve the full diff of a branch against its base branch, or `client.get_diff_summary()` for the list of changed nodes; both use the `DiffTree` GraphQL query. ([#325](https://github.com/opsmill/infrahub-sdk-python/issues/325)) + +### Added + +- Added the Infrahub deployment ID to the `infrahubctl info` command output and a `get_server_information()` method (returning the server version and deployment ID) on the async and sync clients. ([#1017](https://github.com/opsmill/infrahub-sdk-python/issues/1017)) +- Added transparent retry of HTTP 429 (rate-limited) responses on both `InfrahubClient` and `InfrahubClientSync`. Retries use jittered exponential backoff and honour a server-provided `Retry-After` header (delta-seconds or HTTP-date). The behaviour is tunable through four new `Config` fields (`rate_limit_retry_enabled`, `rate_limit_max_retries`, `rate_limit_backoff_base`, `rate_limit_backoff_max`), and a new `RateLimitError` exception is raised when retries are exhausted. ([#1124](https://github.com/opsmill/infrahub-sdk-python/issues/1124)) +- Added support for tagging SDK requests with a priority via a new `X-Priority` header. A `Priority` enum (`high`, `medium`, `low`) is available from `infrahub_sdk.constants`; set `Config.priority` (env var `INFRAHUB_PRIORITY`) for a client-wide default emitted on every request, or pass `priority=` to individual operations to override it per request. When unset, no header is sent. Works identically on `InfrahubClient` and `InfrahubClientSync`. ([#1151](https://github.com/opsmill/infrahub-sdk-python/issues/1151)) +- Add `infrahubctl schema format` command, an opinionated offline formatter that normalises the key ordering of schema files. Optional flags can also strip redundant default values (`--strip-defaults`), sort attributes/relationships by `order_weight` (`--sort-by-order-weight`), and backfill a missing `order_weight` (`--backfill-order-weight`). +- Added `retry()` and `cancel()` methods to the task manager. The `Task` model now exposes `available_actions` along with `can_retry` / `can_cancel` helpers. +- Added an opt-in `include_diagnostics` flag to the task manager's `all()`, `filter()`, and `get()` methods. When enabled, tasks expose an `error` field, and `webhook-send` tasks are returned as `WebhookDeliveryTask` instances carrying `http_request` / `http_response` delivery details. +- Added support for the new `IPAddress` attribute kind. Values are exposed as bare `ipaddress.IPv4Address`/`IPv6Address` objects (no prefix) and serialized to a bare-address string when writing, alongside the existing `IPHost` and `IPNetwork` kinds. +- The JSON schema generated for `.infrahub.yml` now warns when a definition has not said what it depends on, so YAML language servers flag it while the file is being edited. A Python transform or generator definition with no `watch` block is flagged, and so is a `watch` value that is not a mapping, including the bare `watch:` that parses as null and records nothing. Both warnings are advisory only: the models still accept every one of those forms. An empty `watch: {}` or `files: []` stays clean, since either one records that the author checked and nothing beyond what Infrahub detects needs watching. The generated schema also picks up the `watch` block on generator definitions, which it was previously rejecting as an unknown property. +- The request priority (`X-Priority` header) can now be carried on the client's `RequestContext` via a new `priority` field, alongside the existing client-wide `Config.priority` default and per-call `priority=` override. Resolution precedence is per-call `priority=` > `request_context.priority` > `Config.priority` > no header. The priority is emitted as a header only and is never included in the mutation body. Works identically on `InfrahubClient` and `InfrahubClientSync`. +- Import `pyarrow` lazily in the line-delimited JSON importer so that `infrahubctl` commands other than `object load` no longer require the `ctl` extra (and its heavy `pyarrow` dependency) to be installed. + +### Changed + +- Paginated queries generated by `all()`, `filters()`, `get()` and resource pool allocation lookups now pass `offset` and `limit` as GraphQL variables instead of inlining them in the query text. The query document stays identical across pages, allowing the Infrahub server to reuse its cached query analysis, and the query is now rendered once per call instead of once per page. `generate_query_data` also accepts variable placeholder strings (for example `"$offset"`) for its `offset` and `limit` arguments. +- Raised the default `Config.rate_limit_max_retries` from 5 to 10, so a request shed with HTTP 429 keeps retrying (honouring `Retry-After`) for longer before raising `RateLimitError`. This lets background work ride out a longer burst of server-side backpressure. Callers that prefer to give up sooner can lower the value. +- The hand-maintained schema models in `infrahub_sdk.schema` are now backed by the generated write/read contract (`infrahub_sdk.schema.generated`). Public names, import paths, and behavior methods are unchanged, but a few defaults and constraints now match the server contract: + + - `AttributeKind.STRING` has been removed. It was deprecated and `kind="String"` was already rejected server-side; use `AttributeKind.TEXT` instead. + - Write and read models drop unknown fields silently (`extra="ignore"`). A submitted field that is not part of the write contract — read-level, internal, or a typo — is dropped rather than rejected, and a read model tolerates additional fields returned by a newer server. + - Write-model defaults now match the server contract: relationship `min_count`/`max_count` default to `0` (was `None`), node `branch` defaults to `"aware"`, `generate_profile` defaults to `True`, and `generate_template` defaults to `False`. This changes the round-trip output of programmatically-built schemas. + + Constructing `AttributeSchema(name=..., kind=AttributeKind.TEXT, ...)`, `NodeSchema`, `GenericSchema`, `RelationshipSchema`, `SchemaRoot`, and the read-side `*API` models continues to work unchanged. + ## [1.22.3](https://github.com/opsmill/infrahub-sdk-python/tree/v1.22.3) - 2026-08-19 ### Fixed diff --git a/dev/specs/ihs-249-sdk-429-retry/alignment-check.md b/dev/specs/ihs-249-sdk-429-retry/alignment-check.md new file mode 100644 index 000000000..5caf8a315 --- /dev/null +++ b/dev/specs/ihs-249-sdk-429-retry/alignment-check.md @@ -0,0 +1,43 @@ +# Spec/Ask Alignment Check: SDK retry with backoff on HTTP 429 responses + +**Date**: 2026-07-07 +**Feature**: [spec.md](./spec.md) + +## 1. Source + +**Source PRD**: Jira IHS-249 — "SDK retry with backoff on HTTP 429 responses" +(`https://opsmill.atlassian.net/browse/IHS-249`), fetched via the Atlassian MCP tool. +The issue body is itself a full, structured PRD (Problem Statement, Solution Overview, 9 User +Stories, 3 prioritised User Journeys with acceptance criteria, FR-001…009, Key Entities, Edge +Cases, SC-001…005, Implementation/Testing Decisions, Out of Scope, one Open Question). Related +GitHub issue: opsmill/infrahub-sdk-python#1124. No secondary URLs to fetch. + +## 2. Verdict + +Result: ✅ ALIGNED + +`spec.md` faithfully carries every PRD requirement, acceptance criterion, and scope boundary. +The only additions are an expansion of an existing requirement and the authorized resolution of +the PRD's explicit open question — neither is drift under the check's definition. + +## 3. Findings + +| Severity | Category | PRD reference | Spec reference | Description | +| ---------- | ---------- | --------------- | ---------------- | ------------- | +| ✅ none | missing | FR-001…009 | FR-001…009 | All nine functional requirements present, none dropped or softened (attempt cap, jittered+clamped backoff, Retry-After both forms, malformed fallback, RateLimitError with url/attempts/last-Retry-After, all request paths, per-retry logging, async/sync parity, tune+disable). | +| ✅ none | missing | Journeys P1–P3, User Stories 1–9 | US1–US4, Edge Cases | P1/P2/P3 journeys map to US1/US2/US3; PRD user story 8 (tune/disable) surfaced as US4. All acceptance scenarios preserved. | +| ✅ none | missing | SC-001…005 | SC-001…005 | Success criteria carried over with equivalent semantics. | +| ✅ none | contradicted | Out of Scope (503, server-side INFP-636/635, `retry_on_failure`) | Out of Scope | Scope boundaries reproduced verbatim; nothing contradicted. | +| ℹ️ info | added (authorized) | Open Question (chain httpx.HTTPStatusError as `__cause__`?) | FR-005, Assumptions | The PRD's single open question was resolved affirmatively (chain the transport error as `__cause__`). The parent prep flow explicitly authorizes autonomous clarification resolution; recorded as an assumption. Not drift. | +| ℹ️ info | added (derived) | FR-009 (disable via Config) | SC-006 | Spec adds SC-006 (disabled path raises immediately). This is a measurable expansion of FR-009, not new scope. | +| ℹ️ info | added (design) | Assumption: single `_request` chokepoint | plan.md R1 / data-model | Plan (not spec) records that multipart/streaming bypass `_request`, so retry is applied at three sites. This corrects a PRD *assumption* at the implementation layer while still satisfying FR-006; spec requirements unchanged. Not spec drift. | + +No requirements are missing, no acceptance criteria dropped or softened, no requirement semantics +changed, and no off-scope scope items were introduced. The Config field defaults (enabled, 5, 0.5, +60), the new `RateLimitError`, and the additive-only API surface all match the PRD exactly. + +## 4. Action + +**Proceed.** No remediation passes required (remediation counter: 0). `tasks.md` is safe to hand to +the implementation phase. The affirmative resolution of the open question and the SC-006 derivation +are documented above for traceability. diff --git a/dev/specs/ihs-249-sdk-429-retry/checklists/requirements.md b/dev/specs/ihs-249-sdk-429-retry/checklists/requirements.md new file mode 100644 index 000000000..31047f535 --- /dev/null +++ b/dev/specs/ihs-249-sdk-429-retry/checklists/requirements.md @@ -0,0 +1,39 @@ +# Specification Quality Checklist: SDK retry with backoff on HTTP 429 responses + +**Purpose**: Validate specification completeness and quality before proceeding to planning +**Created**: 2026-07-07 +**Feature**: [spec.md](../spec.md) + +## Content Quality + +- [x] No implementation details (languages, frameworks, APIs) +- [x] Focused on user value and business needs +- [x] Written for non-technical stakeholders +- [x] All mandatory sections completed + +## Requirement Completeness + +- [x] No [NEEDS CLARIFICATION] markers remain +- [x] Requirements are testable and unambiguous +- [x] Success criteria are measurable +- [x] Success criteria are technology-agnostic (no implementation details) +- [x] All acceptance scenarios are defined +- [x] Edge cases are identified +- [x] Scope is clearly bounded +- [x] Dependencies and assumptions identified + +## Feature Readiness + +- [x] All functional requirements have clear acceptance criteria +- [x] User scenarios cover primary flows +- [x] Feature meets measurable outcomes defined in Success Criteria +- [x] No implementation details leak into specification + +## Notes + +- The PRD's single open question (whether the exhaustion error should chain the + underlying transport error as its cause) was resolved affirmatively and encoded + into FR-005 and the Assumptions section, so no [NEEDS CLARIFICATION] markers remain. +- Entity names in the spec are described in capability terms (e.g. "rate-limit retry + decision logic") rather than concrete class names to keep the spec implementation-agnostic; + concrete names (`RateLimitRetryHandler`, `RateLimitError`, `Config` fields) are deferred to plan.md. diff --git a/dev/specs/ihs-249-sdk-429-retry/contracts/config.md b/dev/specs/ihs-249-sdk-429-retry/contracts/config.md new file mode 100644 index 000000000..33897e1de --- /dev/null +++ b/dev/specs/ihs-249-sdk-429-retry/contracts/config.md @@ -0,0 +1,37 @@ +# Contract: `Config` rate-limit fields (additive, public) + +Added to `infrahub_sdk/config.py::ConfigBase`. All additive; no existing field changes. + +```python +rate_limit_retry_enabled: bool = Field( + default=True, + description="Retry requests that receive HTTP 429 using backoff. Set False to disable.", +) +rate_limit_max_retries: int = Field( + default=5, + ge=0, + description="Maximum number of retries after the initial attempt when receiving HTTP 429.", +) +rate_limit_backoff_base: float = Field( + default=0.5, + gt=0, + description="Base interval in seconds for exponential backoff between 429 retries.", +) +rate_limit_backoff_max: float = Field( + default=60.0, + gt=0, + description="Maximum wait in seconds for any single 429 retry (also clamps Retry-After).", +) +``` + +## Backward compatibility + +- Purely additive; existing code constructing `Config(...)` / `InfrahubClient(...)` is unaffected. +- Environment-variable overrides follow the existing `BaseSettings` mechanism (e.g. + `INFRAHUB_RATE_LIMIT_MAX_RETRIES`), consistent with current fields. + +## Guarantees + +- `rate_limit_retry_enabled=False` ⇒ a 429 is returned/raised exactly as before this feature + (no wait, no extra attempt). (FR-009, SC-006) +- Defaults produce transparent retry for typical background workloads. (FR-001) diff --git a/dev/specs/ihs-249-sdk-429-retry/contracts/rate_limit_error.md b/dev/specs/ihs-249-sdk-429-retry/contracts/rate_limit_error.md new file mode 100644 index 000000000..c3e20c176 --- /dev/null +++ b/dev/specs/ihs-249-sdk-429-retry/contracts/rate_limit_error.md @@ -0,0 +1,41 @@ +# Contract: `RateLimitError` (new public exception) + +Added to `infrahub_sdk/exceptions.py`. Subclass of the base `Error`. + +```python +class RateLimitError(Error): + def __init__( + self, + url: str, + attempts: int, + retry_after: float | None = None, + message: str | None = None, + ) -> None: + self.url = url + self.attempts = attempts + self.retry_after = retry_after + if message is None: + message = ( + f"Request to {url} was rate-limited (HTTP 429) after {attempts} attempt(s)." + ) + super().__init__(message) +``` + +## Contract + +- **Raised**: by the client retry driver when 429s persist past `rate_limit_max_retries`. (FR-005) +- **Type**: `isinstance(err, Error)` is `True` — callers catching the SDK base `Error` still catch it. +- **Distinct**: it is NOT an `httpx.HTTPStatusError`; callers can `except RateLimitError` to + distinguish rate-limit exhaustion from other HTTP failures. (User story 6) +- **Attributes**: `url: str`, `attempts: int` (= `max_retries + 1`), `retry_after: float | None` + (last observed `Retry-After` in seconds, `None` if never present or unparseable). +- **Cause chaining**: raised with `raise RateLimitError(...) from http_status_error`, so + `err.__cause__` is the underlying `httpx.HTTPStatusError` built from the final 429 response. + Callers can inspect `err.__cause__.response` for the raw response. (Open-question resolution) + +## Behavioural change (changelog callout) + +Before this feature, a persistent 429 surfaced as `httpx.HTTPStatusError` (via +`raise_for_status()`). With retry enabled (default), it now surfaces as `RateLimitError` +after exhaustion. Callers relying on catching `httpx.HTTPStatusError` for 429 should either +catch `RateLimitError` or inspect `__cause__`. diff --git a/dev/specs/ihs-249-sdk-429-retry/contracts/rate_limit_retry_handler.md b/dev/specs/ihs-249-sdk-429-retry/contracts/rate_limit_retry_handler.md new file mode 100644 index 000000000..aedd7b0e3 --- /dev/null +++ b/dev/specs/ihs-249-sdk-429-retry/contracts/rate_limit_retry_handler.md @@ -0,0 +1,46 @@ +# Contract: `RateLimitRetryHandler` (new, pure logic) + +New module `infrahub_sdk/rate_limit.py`. No I/O, no sleeping — deterministic and unit-testable. + +```python +class RateLimitRetryHandler: + def __init__(self, max_retries: int, backoff_base: float, backoff_max: float) -> None: ... + + def parse_retry_after( + self, header: str | None, *, now: datetime | None = None + ) -> float | None: + """Return seconds to wait per Retry-After, or None if absent/malformed. + - delta-seconds: int(header) + - HTTP-date: (parsedate_to_datetime(header) - now).total_seconds(), floored at 0 + - anything unparseable: None (caller falls back to computed backoff).""" + + def compute_backoff(self, attempt: int) -> float: + """Deterministic exponential ceiling: min(backoff_max, backoff_base * 2**attempt).""" + + def jittered_delay(self, ceiling: float) -> float: + """Full jitter: random.uniform(0, ceiling).""" + + def next_delay( + self, attempt: int, retry_after_header: str | None = None, *, now: datetime | None = None + ) -> float: + """Honour Retry-After if parseable (clamped to backoff_max), else jittered backoff.""" + + def should_retry(self, attempts_made: int) -> bool: + """True while retries remain: attempts_made <= max_retries.""" +``` + +## Contract guarantees (map to FR / SC) + +- `compute_backoff` is monotonic non-decreasing in `attempt` and never exceeds `backoff_max`. (FR-002, SC-003) +- `jittered_delay(c)` ∈ `[0, c]`; two calls (or two handler instances) are extremely unlikely to + match, satisfying "differ between instances". Tests assert jitter by sampling. (SC-003) +- `next_delay` clamps every result — computed *and* `Retry-After` — to `backoff_max`. (FR-003) +- `parse_retry_after` never raises on bad input; returns `None`. (FR-004) +- Past HTTP-date ⇒ `parse_retry_after` returns `0.0`, never negative. (Edge case) +- `should_retry` yields exactly `max_retries` retries ⇒ `max_retries + 1` total sends. (FR-001, SC-004) + +## Determinism for tests + +- `parse_retry_after`/`next_delay` accept an injectable `now` for HTTP-date tests. +- Jitter is the only nondeterministic element; tests either assert on `compute_backoff` + (deterministic ceiling) or assert `0 <= jittered_delay(c) <= c` and that a sample of draws varies. diff --git a/dev/specs/ihs-249-sdk-429-retry/critiques/critique-20260707-111502.md b/dev/specs/ihs-249-sdk-429-retry/critiques/critique-20260707-111502.md new file mode 100644 index 000000000..f26e0c4b5 --- /dev/null +++ b/dev/specs/ihs-249-sdk-429-retry/critiques/critique-20260707-111502.md @@ -0,0 +1,168 @@ +# Critique Report: SDK retry with backoff on HTTP 429 responses + +**Date**: 2026-07-07 +**Feature**: [spec.md](../spec.md) +**Plan**: [plan.md](../plan.md) +**Verdict**: ⚠️ PROCEED WITH UPDATES + +--- + +## Executive Summary + +The spec and plan are strong: the problem is well-evidenced (INFP-636, issue #1124), the +scope is tightly bounded, requirements are testable, and the plan already caught the most +important structural risk — that the "single `_request` chokepoint" assumed by the PRD is +actually three send sites (`_request`, `_request_multipart`, `_get_streaming`). The pure +`RateLimitRetryHandler` / thin-driver split is clean and testable. One genuine correctness +risk was surfaced that must be resolved before implementation: **retrying a multipart upload +can re-send an already-consumed file body**, silently uploading empty/truncated data. That is +the single 🎯 Must-Address. The remaining findings are low-risk hardening (don't log secrets, +document worst-case cumulative wait, add a multipart-body regression test) applied inline. + +--- + +## Product Lens Findings 🎯 + +### Problem Validation + +| ID | Severity | Finding | Suggestion | +| ---- | ---------- | --------- | ------------ | +| P1 | ✅ | Problem is clear and evidenced (background workloads are the traffic most likely rate-limited; callers currently hand-roll retries). No gap. | None. | + +### User Value Assessment + +| ID | Severity | Finding | Suggestion | +| ---- | ---------- | --------- | ------------ | +| P2 | ✅ | Every user story maps to value; MVP is cleanly P1 (transparent retry-and-succeed). | None. | + +### Alternative Approaches + +| ID | Severity | Finding | Suggestion | +| ---- | ---------- | --------- | ------------ | +| P3 | 💡 | The plan doesn't record *why* a custom handler beats off-the-shelf options (`tenacity`, httpx transport-level `retries`). httpx transport retries are connection-level only (not status-code aware) and `tenacity` is a new dependency (out of scope). | Add one line to research.md for the record so reviewers don't re-litigate. (Applied.) | + +### Edge Cases & UX + +| ID | Severity | Finding | Suggestion | +| ---- | ---------- | --------- | ------------ | +| P4 | 💡 | Worst-case cumulative blocking time (~`max_retries × backoff_max` ≈ 300 s with defaults) is bounded but undocumented; an interactive caller could block ~5 min. | Document the worst-case total wait and note interactive callers can lower `rate_limit_max_retries`/`rate_limit_backoff_max` or disable. (Applied to plan.) | + +### Success Measurement + +| ID | Severity | Finding | Suggestion | +| ---- | ---------- | --------- | ------------ | +| P5 | ✅ | SC-001..006 are measurable and mapped to acceptance scenarios and quickstart validations. | None. | + +--- + +## Engineering Lens Findings 🔬 + +### Architecture Soundness + +| ID | Severity | Finding | Suggestion | +| ---- | ---------- | --------- | ------------ | +| E1 | ✅ | Pure handler + thin async/sync drivers is the right shape; single logic contract satisfies FR-008. Multi-site chokepoint already documented (research R1). | None. | + +### Failure Mode Analysis + +| ID | Severity | Finding | Suggestion | +| ---- | ---------- | --------- | ------------ | +| E2 | 🎯 | **Multipart retry can re-send a consumed body.** `_request_multipart` receives a `files` dict that may hold open file handles / streams. On the first send httpx reads them to EOF; a retry re-sends the *same* handles, uploading empty or truncated data — a silent data-corruption bug that only manifests under rate-limiting. | The multipart send site MUST rewind (`seek(0)`) or re-materialize the payload before each retry, or the driver must accept a payload *factory* that produces a fresh body per attempt. Add this constraint to plan/data-model and a dedicated task + regression test. (Applied.) | +| E3 | 🤔 | Retrying mutations relies on the PRD assumption that a 429 is always a pre-processing rejection (no partial write). If the server ever emits 429 after partial processing, a retried POST double-writes. | Accept the PRD's explicit assumption (rate-limit 429 = pre-processing) for this scope; recorded as an assumption in spec.md. Revisit only if server semantics change. (Resolved — no change.) | + +### Security & Privacy + +| ID | Severity | Finding | Suggestion | +| ---- | ---------- | --------- | ------------ | +| E4 | 💡 | Retry logging must not leak secrets. The login/refresh paths carry `Authorization: Bearer …` headers and username/password payloads; logging is spec'd to include only URL/attempt/delay, but this should be stated as an explicit constraint so it isn't broadened later. | State in research R8 that retry logs MUST include only URL, attempt number, and delay — never headers or payload. (Applied.) | + +### Performance & Scalability + +| ID | Severity | Finding | Suggestion | +| ---- | ---------- | --------- | ------------ | +| E5 | ✅ | Full jitter de-correlates concurrent clients (thundering-herd mitigation). No hot paths introduced. | None. | + +### Testing Strategy + +| ID | Severity | Finding | Suggestion | +| ---- | ---------- | --------- | ------------ | +| E6 | 💡 | Test matrix is comprehensive but has no guard for E2 (multipart body re-read). | Add a client-level test: a multipart upload that gets 429→200 must re-send the *full* body on the retry (assert bytes received on attempt 2 equal attempt 1). (Applied to quickstart + will be a task.) | + +### Operational Readiness + +| ID | Severity | Finding | Suggestion | +| ---- | ---------- | --------- | ------------ | +| E7 | ✅ | WARNING-level per-retry logging via the existing module logger is appropriate for a library; host apps control handlers. | None. | + +### Dependencies & Integration + +| ID | Severity | Finding | Suggestion | +| ---- | ---------- | --------- | ------------ | +| E8 | ✅ | No new dependencies (httpx + stdlib). `retry_on_failure` left untouched (out of scope). Additive Config + one exception; behavioural break documented for changelog. | None. | + +--- + +## Cross-Lens Insights 🔗 + +| ID | Finding | Product Impact | Engineering Impact | Suggestion | +| ---- | --------- | --------------- | ------------------- | ------------ | +| X1 | Multipart body re-read (E2) | A "successful" upload that silently uploaded nothing is worse than a visible failure — directly harms the P1 "transparent success" promise. | Silent data corruption under load; hard to diagnose. | Make re-readable payload a hard requirement before implementation. (Applied.) | + +--- + +## Findings Summary + +| Metric | Count | +| -------- | ------- | +| 🎯 Must-Address | 1 | +| 💡 Recommendations | 4 | +| 🤔 Questions | 1 (resolved inline) | +| Product findings | 5 | +| Engineering findings | 8 | +| Cross-lens findings | 1 | + +--- + +## Consolidated Findings Table + +| ID | Lens | Severity | Category | Finding | Suggestion | +| ---- | ------ | ---------- | ---------- | --------- | ------------ | +| E2/X1 | Both | 🎯 | Failure Modes × UX | Multipart retry re-sends consumed body | Require rewind/re-materialize payload per attempt + regression test | +| P3 | Product | 💡 | Alternatives | No record of why custom vs tenacity/httpx retries | One line in research.md | +| P4 | Product | 💡 | Edge/UX | Worst-case cumulative wait undocumented | Document ~max_retries×backoff_max; tuning guidance | +| E4 | Engineering | 💡 | Security | Risk of logging secrets | Constrain logs to URL/attempt/delay only | +| E6 | Engineering | 💡 | Testing | No guard for multipart body re-read | Add multipart full-body-on-retry test | +| E3 | Engineering | 🤔 | Failure Modes | Retrying mutations assumes pre-processing 429 | Accept PRD assumption; recorded | + +--- + +## Recommended Actions + +### 🎯 Must-Address (Before Proceeding) + +1. **E2/X1**: Add to `plan.md` (Key design decisions) and `data-model.md` (retry driver) the + requirement that the multipart send site rewinds or re-materializes its payload before each + retry attempt; ensure `tasks.md` includes a task and a regression test for it. + +### 💡 Recommendations (Strongly Suggested) + +1. **P3**: Record the build-vs-buy rationale in `research.md`. +2. **P4**: Document the worst-case cumulative wait and tuning guidance in `plan.md`. +3. **E4**: State the log-content constraint (no headers/payload) in `research.md` R8. +4. **E6**: Add the multipart full-body-on-retry validation to `quickstart.md`. + +### 🤔 Questions (Need Stakeholder Input) + +1. **E3**: Confirmed resolved by accepting the PRD's explicit "429 is pre-processing" assumption; no blocker. + +--- + +**Severity Legend**: + +- 🎯 **Must-Address**: Blocks proceeding to implementation +- 💡 **Recommendation**: Strongly suggested improvement +- 🤔 **Question**: Needs stakeholder input to resolve + +--- + +*Generated by `/speckit.critique` — Dual-lens strategic and technical review for spec-driven development.* diff --git a/dev/specs/ihs-249-sdk-429-retry/data-model.md b/dev/specs/ihs-249-sdk-429-retry/data-model.md new file mode 100644 index 000000000..9b4b6aa5d --- /dev/null +++ b/dev/specs/ihs-249-sdk-429-retry/data-model.md @@ -0,0 +1,83 @@ +# Data Model: SDK retry with backoff on HTTP 429 responses + +This feature is behavioural, not persistence-oriented. The "entities" are the config +fields, the pure decision helper, and the exception. + +## Config fields (added to `ConfigBase`) + +| Field | Type | Default | Meaning | Validation | +| ------- | ------ | --------- | --------- | ------------ | +| `rate_limit_retry_enabled` | `bool` | `True` | Master on/off switch for 429 retry (FR-009). | — | +| `rate_limit_max_retries` | `int` | `5` | Max number of *retries* after the initial attempt; total sends = value + 1 (FR-001, SC-004). | `>= 0` | +| `rate_limit_backoff_base` | `float` | `0.5` | Base interval (seconds) for exponential backoff (FR-002). | `> 0` | +| `rate_limit_backoff_max` | `float` | `60.0` | Ceiling (seconds) for any single wait, incl. `Retry-After` (FR-002, FR-003). | `> 0` | + +Notes: + +- Fields live on `ConfigBase` so `Config` and any subclass inherit them. +- `rate_limit_max_retries = 0` with retry enabled means: one attempt, and a 429 immediately + raises `RateLimitError` (0 retries) — distinct from disabled, which raises the raw error. + +## `RateLimitRetryHandler` (new, pure / I/O-free) — `infrahub_sdk/rate_limit.py` + +Owns all decision logic; performs no sleeping and no network I/O. + +**Construction**: `RateLimitRetryHandler(max_retries: int, backoff_base: float, backoff_max: float)`. + +**State**: none required beyond config values; the current attempt count is passed in per call +(keeps the handler reusable and thread/async-safe). + +**Behaviour**: + +| Method | Signature (conceptual) | Returns | Rules | +| -------- | ------------------------ | --------- | ------- | +| `parse_retry_after` | `(header: str \| None, *, now=…) -> float \| None` | seconds, or `None` | delta-seconds → `int`; HTTP-date → `(date-now).total_seconds()` floored at 0; malformed/absent → `None` (FR-003, FR-004, past-date edge case). | +| `compute_backoff` | `(attempt: int) -> float` | ceiling seconds | `min(backoff_max, backoff_base * 2**attempt)` — the deterministic exponential ceiling (used for assertions in SC-003). | +| `jittered_delay` | `(ceiling: float) -> float` | seconds | `random.uniform(0, ceiling)` — full jitter (FR-002, SC-003). | +| `next_delay` | `(attempt: int, retry_after_header: str \| None, *, now=…) -> float` | seconds to wait | If `parse_retry_after` returns a value, use `min(it, backoff_max)`; else `jittered_delay(compute_backoff(attempt))`. All results clamped to `backoff_max`. | +| `should_retry` | `(attempts_made: int) -> bool` | bool | `attempts_made <= max_retries` (i.e. retries remain); see research R7. | + +`attempt` passed to backoff is 0-indexed (first retry uses `attempt=0` → ceiling `backoff_base`). + +## `RateLimitError` (new) — `infrahub_sdk/exceptions.py` + +Subclass of the existing base `Error`. + +| Attribute | Type | Meaning | +| ----------- | ------ | --------- | +| `url` | `str` | The request URL that kept getting rate-limited (FR-005). | +| `attempts` | `int` | Total attempts made before giving up (= `max_retries + 1`). | +| `retry_after` | `float \| None` | The last `Retry-After` value observed (parsed seconds), or `None`. | +| `message` | `str \| None` | Human-readable summary (default built from the above). | +| `__cause__` | `httpx.HTTPStatusError` | The underlying transport error, chained via `raise … from …` (open-question resolution). | + +Constructor: `RateLimitError(url, attempts, retry_after=None, message=None)`. + +## Retry driver (client method, not a standalone entity) + +Two symmetric variants, one per client: + +- Async: `await self._send_with_rate_limit_retry(send, url)` where `send` is an + `async` callable returning `httpx.Response`; sleeps via `asyncio.sleep`. +- Sync: `self._send_with_rate_limit_retry(send, url)` where `send` is a sync callable; + sleeps via `time.sleep`. + +Loop (identical logic both variants, FR-008): + +1. If `not config.rate_limit_retry_enabled` → `return send()` (single attempt, FR-009). +2. `attempts = 0`; loop: `response = send()`; `attempts += 1`. +3. If `response.status_code != 429` → return `response`. +4. If handler says no retries remain → build `httpx.HTTPStatusError` from the response and + `raise RateLimitError(url, attempts, last_retry_after) from http_error` (FR-005). +5. Else compute `delay = handler.next_delay(attempt=attempts-1, retry_after_header=…)`, + log `WARNING` (url, attempt, delay) (FR-007), sleep `delay`, continue. + +**`send` callable contract (critique E2/X1 — Must-Address).** Because `send` is invoked once +per attempt, it MUST yield a fully-readable request body on every call: + +- `_request` (JSON payload): the dict is re-serialized per send — inherently safe. +- `_request_multipart`: the driver MUST rewind each file object (`seek(0)`) or materialize the + body to bytes before each attempt, so a retried upload carries the complete body rather than a + stream already consumed to EOF on the first attempt. A regression test asserts the retried + upload's body equals the first attempt's body. +- `_get_streaming` initiation: retried only before any body is read, so no re-read hazard. diff --git a/dev/specs/ihs-249-sdk-429-retry/opsmill-implement-report.md b/dev/specs/ihs-249-sdk-429-retry/opsmill-implement-report.md new file mode 100644 index 000000000..c680c12a3 --- /dev/null +++ b/dev/specs/ihs-249-sdk-429-retry/opsmill-implement-report.md @@ -0,0 +1,84 @@ +# Opsmill Implement Report: SDK retry with backoff on HTTP 429 responses + +**Status**: ✅ DONE + +## 1. Header + +- **Feature**: SDK retry with backoff on HTTP 429 responses (Jira IHS-249, GitHub #1124) +- **Spec dir**: `specs/ihs-249-sdk-429-retry/` (real path `dev/specs/ihs-249-sdk-429-retry/`, via the `specs → dev/specs` symlink) +- **Base commit**: `a55cbaa` (prep artifacts, pre-implementation) +- **Head commit**: `8917fb9` +- **Commits produced** (7): `c71238c` (foundational machinery) → `ed0457c` (US1) → `32abfd5` (US2) → `d309335` (US3) → `1727b96` (US4) → `d68d734` (polish) → `8917fb9` (review fixes) +- **Wall-clock**: implement loop + review ≈ 55 min of subagent runtime (6 impl chunks + 3 review agents + 1 fix agent). + +## 2. Chunk-by-chunk ledger + +| # | Chunk (phase) | Tasks | ✅ | ⚠️ | ❌ | Commit | Notes flagged upward | +| --- | --------------- | ------- | ---- | ---- | ---- | -------- | ---------------------- | +| 1 | Phase 1+2 Setup + Foundational | T001–T010 (10) | 10 | 0 | 0 | `c71238c` | Streaming retry uses `ExitStack`/`AsyncExitStack`; failed 429 stream read+closed, successful stream left open until caller done. Logs only URL/attempt/delay (no secrets). No client tests here (deferred to later chunks). | +| 2 | Phase 3 US1 | T011, T012 (2) | 2 | 0 | 0 | `ed0457c` | T012 needed no `client.py` change — the `_request` retry + non-429 passthrough already correct. | +| 3 | Phase 4 US2 | T013 (1) | 1 | 0 | 0 | `32abfd5` | HTTP-date case asserted with an inclusive time window (driver parses against `datetime.now`); fixed-form cases assert exactly. | +| 4 | Phase 5 US3 | T014, T015 (2) | 2 | 0 | 0 | `d309335` | Scripted 429 responses attach `request=httpx.Request(...)` so `raise_for_status()` yields `HTTPStatusError` (test fabrication; driver correct). T015 satisfied without code change. | +| 5 | Phase 6 US4 | T016 (1) | 1 | 0 | 0 | `1727b96` | Disabled path asserted at `_request` level (raw 429 returned, no `RateLimitError`). Parity compared with a deterministic `Retry-After: 5` (exact cross-client comparison, no jitter noise). | +| 6 | Phase 7 Polish | T017–T021 (5) | 4 | 1 | 0 | `d68d734` | T020 ⚠️: `docs-generate` regenerated 10 UNRELATED files with pre-existing drift that also fail markdownlint (broken `--fix` referencing a missing `.markdownlint.yaml`); committed only the feature's `config.mdx`. Multipart rewind noted as defensive (httpx rewinds seekable files itself). | + +All 21 tasks are `[X]` in `tasks.md`. + +## 3. Tasks not completed + +None. All T001–T021 completed and ticked. + +## 4. Local-pass evidence (REQUIRED) + +Runner: `uv run pytest`. Environment for every row: **Python 3.12.13, pytest 9.0.3, pytest-httpx 0.36.0, asyncio mode=AUTO, project `.venv` (`uv sync --all-groups --all-extras`); no external infrastructure required** (all tests run locally; no E2E deferred). Rows are grouped by test function; the bracketed count is the number of parametrized variants, all PASSED. + +| Test id | Type | Run command | Passed at (ISO 8601) | Env context | Verbatim pass line | +| --------- | ------ | ------------- | ---------------------- | ------------- | -------------------- | +| `test_rate_limit.py` handler suite (17 tests: compute_backoff growth/clamp, jittered_delay bounds/variance, parse_retry_after delta/http-date/past-zero/malformed×5, next_delay honour/fallback/clamp, should_retry budget×2) | unit (pure) | `uv run pytest tests/unit/test_rate_limit.py tests/unit/sdk/test_rate_limit_retry.py -v` | 2026-07-07T12:20:50Z | as above | `17 passed in 0.02s` | +| `test_request_retries_429_then_succeeds` [standard, sync] | unit (client) | `uv run pytest tests/unit/sdk/test_rate_limit_retry.py -v` | 2026-07-07T12:24:28Z | as above | `6 passed in 0.02s` | +| `test_request_passes_non_429_through_untouched` [standard-200, standard-500, sync-200, sync-500] | unit (client) | `uv run pytest tests/unit/sdk/test_rate_limit_retry.py -v` | 2026-07-07T12:24:28Z | as above | `6 passed in 0.02s` | +| `test_request_honours_retry_after` [±delta/http-date/zero/past/above-max × standard+sync = 10] | unit (client) | `uv run pytest tests/unit/sdk/test_rate_limit_retry.py tests/unit/test_rate_limit.py -p no:randomly` | 2026-07-07T12:28:31Z | as above | `35 passed in 0.04s` | +| `test_request_malformed_retry_after_falls_back_to_backoff` [standard, sync] | unit (client) | `uv run pytest tests/unit/sdk/test_rate_limit_retry.py ... -p no:randomly` | 2026-07-07T12:28:31Z | as above | `35 passed in 0.04s` | +| `test_request_exhausts_retries_and_raises_rate_limit_error` [standard, sync] | unit (client) | `uv run pytest tests/unit/sdk/test_rate_limit_retry.py tests/unit/test_rate_limit.py` | 2026-07-07T12:34:00Z | as above | `37 passed in 0.05s` | +| `test_request_disabled_surfaces_raw_429_without_retry` [standard, sync] | unit (client) | `uv run pytest tests/unit/sdk/test_rate_limit_retry.py -v` | 2026-07-07T12:37:14Z | as above | `30 passed in 0.06s` | +| `test_request_max_retries_controls_attempt_count` [standard-0/1/3, sync-0/1/3] | unit (client) | `uv run pytest tests/unit/sdk/test_rate_limit_retry.py -v` | 2026-07-07T12:37:14Z | as above | `30 passed in 0.06s` | +| `test_async_sync_parity_on_identical_429_sequence` [retry-after-then-success, retry-after-exhaust] | unit (client) | `uv run pytest tests/unit/sdk/test_rate_limit_retry.py -v` | 2026-07-07T12:37:14Z | as above | `30 passed in 0.06s` | +| `test_all_request_paths_retry_429_then_succeed` [{standard,sync}×{regular,multipart,streaming} = 6] | unit (client, httpx_mock) | `uv run pytest tests/unit/sdk/test_rate_limit_retry.py::test_all_request_paths_retry_429_then_succeed tests/unit/sdk/test_rate_limit_retry.py::test_multipart_body_survives_retry -v` | 2026-07-07T12:42:57Z | as above | `8 passed` | +| `test_multipart_body_survives_retry` [standard, sync] | unit (client, httpx_mock) | (same as row above) | 2026-07-07T12:42:57Z | as above | `8 passed` | +| `test_parse_retry_after_negative_delta_floored_to_zero` | unit (pure) | `uv run pytest tests/unit/test_rate_limit.py tests/unit/sdk/test_rate_limit_retry.py` | 2026-07-07T13:01:20Z | as above | `62 passed in 0.13s` | +| `test_parse_retry_after_pathological_huge_value_returns_none` | unit (pure) | (same as row above) | 2026-07-07T13:01:20Z | as above | `62 passed in 0.13s` | +| `test_backoff_grows_exponentially_and_clamps` [standard, sync] | unit (client) | (same as row above) | 2026-07-07T13:01:20Z | as above | `62 passed in 0.13s` | +| `test_jitter_differs_between_instances` [standard, sync] | unit (client) | (same as row above) | 2026-07-07T13:01:20Z | as above | `62 passed in 0.13s` | +| `test_rewind_multipart_files_resets_every_file_object` | unit (direct helper) | (same as row above) | 2026-07-07T13:01:20Z | as above | `62 passed in 0.13s` | + +**Final aggregate gate** (orchestrator-run): `uv run pytest tests/unit/test_rate_limit.py tests/unit/sdk/test_rate_limit_retry.py -q` → `62 passed`. No `MISSING` rows; no deferred-E2E rows (feature has no E2E surface — it is a pure client-library behaviour exercised via mocked transports). + +## 5. Review findings + +Reviewed the full diff `a55cbaa..HEAD` with three independent agents (code correctness + async/sync parity + type design; error handling / silent failures; test coverage quality). + +| Severity | File / test | Summary | Disposition | +| ---------- | ------------- | --------- | ------------- | +| HIGH | `rate_limit.py` `parse_retry_after` | Negative `Retry-After` (e.g. `-5`) not floored → sync `time.sleep(-5.0)` raises `ValueError` (crash) + async/sync divergence; negative leaks into `RateLimitError.retry_after`. | **Fixed inline** (`8917fb9`): floor delta-seconds at `0.0`. | +| MEDIUM/HIGH | `rate_limit.py` `parse_retry_after` | Pathological huge digit string raises uncaught `OverflowError` (crash); violates FR-004 fall-back. | **Fixed inline** (`8917fb9`): `except OverflowError: return None`, kept `except ValueError: pass` so HTTP-date fall-through still works. | +| HIGH (test) | `test_rate_limit_retry.py` | SC-003/FR-002 exponential-growth + jitter-divergence unguarded at driver level — a bug pinning `attempt=0` would pass the suite. | **Fixed inline** (`8917fb9`): added `test_backoff_grows_exponentially_and_clamps` (identity-patched jitter → deterministic growth) + `test_jitter_differs_between_instances`, async+sync. | +| HIGH (test) | `test_rate_limit_retry.py` | Multipart regression test passed even if `_rewind_multipart_files` were deleted (httpx rewinds seekable files itself). | **Fixed inline** (`8917fb9`): added direct `test_rewind_multipart_files_resets_every_file_object` that fails if the helper is gutted. | +| LOW | `client.py` `_rewind_multipart_files` | Non-seekable stream retried after 429 would silently send an empty/truncated body (suppressed `seek` error). Real callers pass seekable files, so low impact. | **Deferred** — see §6. | +| LOW | `client.py` `_rewind_multipart_files` | `seek(0)` is forced on the first attempt too (harmless for current fresh-file usage; a subtle change vs. reading from current position). | **Deferred** (no action). | +| LOW | `rate_limit.py` `parse_retry_after` | Fractional `Retry-After: 10.5` → falls back to computed backoff (RFC 7231 defines delta-seconds as integer, so this is spec-compliant). | **No action** (correct per spec). | +| LOW | tests | Untested: `err.retry_after is None` exhaustion path, disabled path on multipart/streaming, Config defaults/validators, explicit auth-path test. | **Deferred** (shared chokepoint / shared guard make these low-risk; recorded for a future hardening pass). | + +## 6. Autonomous decisions + +1. **Chunk merge**: merged Phase 1 (Setup — 2 stub-creation tasks) into Phase 2 (Foundational). Phase 1 only creates empty stubs that Phase 2 immediately fills; running them as separate clean-context subagents would produce a throwaway "empty class" commit. This is a cohesive-seam merge, not merging two independent increments — review granularity is preserved because the foundational machinery is one natural unit. +2. **`docs-generate` scope (T020)**: `uv run invoke docs-generate` regenerated 10 files unrelated to this feature (`client.mdx`, `node/*`, `graph_traversal/*`) reflecting pre-existing docstring drift, and those regenerated files fail markdownlint because the tool's `--fix` step references a missing `.markdownlint.yaml` and silently no-ops. The subagent reverted the 10 unrelated files and committed only the feature-relevant `config.mdx` (the 4 new `rate_limit_*` fields). **Flag for the user**: this docs-tooling breakage (broken markdownlint `--fix`, plus a `docs-validate`-vs-`lint-docs` conflict) is a pre-existing repo issue worth a separate follow-up; and a full clean `docs-generate` commit for the unrelated drift may be wanted independently. +3. **Multipart rewind is defensive**: reviewers confirmed httpx's `FileField.render_data()` already `seek(0)`s seekable files on every render, so `_rewind_multipart_files` is redundant for the seekable file objects real callers pass, and cannot help non-seekable streams. It is kept as harmless defence-in-depth and is now directly unit-guarded, but the original critique E2/X1 concern was largely already mitigated by httpx itself. +4. **`Retry-After` except split**: the code-review's literal suggestion `except (ValueError, OverflowError): return None` would have broken the HTTP-date branch (an HTTP-date fails `int()` with `ValueError` and must fall through to date parsing). Split into `except OverflowError: return None` + `except ValueError: pass` to preserve HTTP-date handling — verified by the still-green HTTP-date tests. +5. **Deferred LOW findings** (§5): recorded rather than fixed, since none affects the shipped behaviour for real callers and fixing them (e.g. surfacing an error on non-seekable-stream retry) is a design choice better made explicitly. + +## 7. Suggested next steps + +1. **Open a PR** for branch `dga/feat-409-retry-ivj0i` (base `stable`) — the feature is complete, tested (62 passing), and reviewed. Ensure both towncrier fragments (`changelog/1124.added.md`, `1124.changed.md`) are included; the `429 → RateLimitError` behaviour change is a caller-visible change flagged in the changed fragment. +2. **Optional hardening** (deferred LOW findings): decide whether a 429 retry on a non-seekable multipart stream should raise a clear error instead of silently sending an empty body; add the small missing tests (`retry_after is None` exhaustion, Config defaults/validators). +3. **Separate follow-up** for the repo's `docs-generate`/markdownlint tooling breakage (missing `.markdownlint.yaml`; unrelated `.mdx` drift) — outside this feature's scope. +4. **Branch-name note**: the branch is `dga/feat-409-retry-ivj0i` (says 409) but the feature is HTTP **429** throughout; a pre-existing branch-name typo, harmless, mentioned so the PR title uses 429. diff --git a/dev/specs/ihs-249-sdk-429-retry/plan.md b/dev/specs/ihs-249-sdk-429-retry/plan.md new file mode 100644 index 000000000..2cfea2f63 --- /dev/null +++ b/dev/specs/ihs-249-sdk-429-retry/plan.md @@ -0,0 +1,148 @@ +# Implementation Plan: SDK retry with backoff on HTTP 429 responses + +**Branch**: `dga/feat-409-retry-ivj0i` | **Date**: 2026-07-07 | **Spec**: [spec.md](./spec.md) + +**Input**: Feature specification from `specs/ihs-249-sdk-429-retry/spec.md` (Jira IHS-249, GitHub #1124) + +## Summary + +Make the SDK transparently retry any request that receives HTTP 429, using jittered +exponential backoff (or the server's `Retry-After` when present, clamped to a max), +and raise a dedicated `RateLimitError` once a configurable retry budget is exhausted. +Behaviour is tunable and fully disableable through `Config`, identical across the async +and sync clients, and covers every request path where a 429 can occur — including the +multipart-upload and streaming-init paths that currently bypass the `_request` method. + +Technical approach: a pure, I/O-free `RateLimitRetryHandler` owns all decision logic +(parse `Retry-After`, compute jittered/clamped backoff, decide continue-vs-exhausted). +Two thin retry drivers on the clients (`async` sleeps with `asyncio.sleep`, `sync` with +`time.sleep`) wrap the existing "send once" call sites and consult the handler. A new +`RateLimitError(Error)` carries `url`, `attempts`, and `last_retry_after`, chaining the +underlying `httpx.HTTPStatusError` as its `__cause__`. + +## Technical Context + +**Language/Version**: Python 3.10–3.13 + +**Primary Dependencies**: httpx (transport), pydantic v2 (Config via pydantic-settings `BaseSettings`); stdlib `random`, `time`, `asyncio`, `email.utils` (HTTP-date parsing), `logging`. No new dependencies. + +**Storage**: N/A + +**Testing**: pytest (`tests/unit/`), with a pluggable `requester` / `sync_requester` on `Config` and mocked httpx transports as prior art. + +**Target Platform**: Cross-platform Python library (async + sync clients) + +**Project Type**: Library (async/sync dual API) — single project layout under `infrahub_sdk/`. + +**Performance Goals**: No throughput target; correctness of the delay sequence and attempt count is what matters. Waits must be observable and clamped; jitter must de-correlate concurrent clients. + +**Constraints**: Must not change existing public method signatures. New behaviour must be default-on but fully disableable. A 429 that previously raised `httpx.HTTPStatusError` will now raise `RateLimitError` after exhaustion — a caller-visible change requiring a changelog callout. + +**Scale/Scope**: Four additive `Config` fields, one new exception, one new pure-logic helper module, and retry drivers wired into three send sites per client (regular request, multipart, streaming-init). + +## Constitution Check + +*GATE: Must pass before Phase 0 research. Re-check after Phase 1 design.* + +The project constitution (`.specify/memory/constitution.md` → `dev/constitution.md`) is an +unfilled template with no ratified principles, so there are no formal gates to evaluate. +The de-facto project standards from `AGENTS.md` are treated as the applicable gates: + +- **Async/sync dual pattern** — SATISFIED: every behaviour is delivered on both `InfrahubClient` and `InfrahubClientSync`, with a shared pure handler so there is one contract to reason about (FR-008). +- **Type hints on all signatures** — SATISFIED: all new functions/methods are fully typed. +- **No new dependencies** — SATISFIED: stdlib + existing httpx only. +- **Do not modify generated code (protocols.py)** — SATISFIED: no generated code touched. +- **Additive public API** — SATISFIED: four new `Config` fields + one new exception; no existing signature changes. The one behavioural break (429 → `RateLimitError`) is documented in the changelog. + +No violations; Complexity Tracking table not required. + +## Project Structure + +### Documentation (this feature) + +```text +specs/ihs-249-sdk-429-retry/ +├── plan.md # This file +├── research.md # Phase 0 output +├── data-model.md # Phase 1 output +├── quickstart.md # Phase 1 output +├── contracts/ # Phase 1 output (public API surface) +│ ├── config.md +│ ├── rate_limit_error.md +│ └── rate_limit_retry_handler.md +├── checklists/ +│ └── requirements.md # From specify phase +└── tasks.md # Phase 2 output (/speckit-tasks — not created here) +``` + +### Source Code (repository root) + +```text +infrahub_sdk/ +├── client.py # InfrahubClient (async, class @ L349) + InfrahubClientSync (sync, class @ L2053). +│ # MODIFY BOTH clients symmetrically — each has the same three send sites: +│ # • async: _request (L1486), _request_multipart (L1383), _get_streaming (L1455) +│ # • sync: _request (L3583), _request_multipart (L2331), _get_streaming (L3524) +│ # Add a retry driver per client (async awaits asyncio.sleep; sync calls time.sleep) +│ # and wire it into all three of that client's send sites. +├── config.py # ConfigBase / Config (pydantic-settings BaseSettings). +│ # MODIFY: add four rate_limit_* fields on ConfigBase. +├── exceptions.py # Error base + subclasses. +│ # MODIFY: add RateLimitError(Error). +└── rate_limit.py # NEW: RateLimitRetryHandler (pure, I/O-free decision logic). + +tests/unit/ +├── test_rate_limit.py # NEW: handler unit tests (backoff, jitter, clamp, Retry-After parse). +└── sdk/ + └── test_rate_limit_retry.py # NEW: client-level tests (429→200, persistent 429→RateLimitError, + # Retry-After honouring, disabled path), parametrized async+sync. +``` + +**Structure Decision**: Single-project library layout. The pure handler lives in a new +`infrahub_sdk/rate_limit.py` (no I/O, unit-testable in isolation). The clients keep their +existing structure; the retry loop is added as a small driver method rather than being +inlined, so the async and sync variants stay symmetric and share the same handler instance +logic. `Config` gains fields on `ConfigBase` so both `Config` and any config subclasses inherit them. + +## Key design decisions + +1. **Chokepoint is not singular.** `login()`/`refresh_login()` route through `_request`, but + `_request_multipart` and `_get_streaming` build their own `httpx.AsyncClient` and bypass + `_request`. To satisfy FR-006 (queries, mutations, multipart, streaming, auth), the retry + driver wraps a "send once → return response" callable and is applied at all three send sites + on each client, not only `_request`. See research.md R1. +2. **Detection point.** `_request` and friends return the raw `httpx.Response` (callers invoke + `raise_for_status()` later). The driver inspects `response.status_code == 429` directly, so + no exception needs to be raised/caught to trigger a retry. On exhaustion the driver raises + `RateLimitError`, chaining the `httpx.HTTPStatusError` produced from the final 429 response. +3. **Streaming semantics.** For `_get_streaming`, only the *initiation* (opening the stream and + reading the response status) is retried; a 429 arrives in the response headers before body + streaming begins, so retry-on-init is safe and matches FR-006's "streaming initiation". +4. **Sleep abstraction.** The pure handler returns a delay (float seconds); the async driver + awaits `asyncio.sleep(delay)` and the sync driver calls `time.sleep(delay)`. The handler + never sleeps, keeping it deterministic and unit-testable. +5. **Disabled path.** When `rate_limit_retry_enabled=False`, the driver performs exactly one + send and returns the response untouched (no 429 inspection, no wait), preserving the exact + pre-feature behaviour (FR-009 / SC-006). +6. **Re-readable payloads on retry (critique E2/X1 — Must-Address).** The driver re-invokes a + "send once" callable per attempt. For JSON payloads (`_request`) this is safe (the dict is + re-serialized each send). For **multipart uploads** (`_request_multipart`) the `files` payload + may contain open file handles / streams that httpx reads to EOF on the first attempt; naively + re-sending them would upload an empty or truncated body. The multipart send site therefore + MUST produce a fresh, fully-readable body per attempt — either by rewinding each file object + (`seek(0)`) before re-sending or by materializing the payload into bytes once and re-sending + those bytes. A regression test MUST assert the retried upload carries the full body. This also + applies to the streaming-init path, which only retries *before* any body is consumed. + +## Operational notes + +- **Worst-case cumulative wait (critique P4).** With retry enabled, the maximum time a call can + block before raising `RateLimitError` is bounded by roughly `rate_limit_max_retries × + rate_limit_backoff_max` (defaults: 5 × 60 s ≈ 300 s), since each wait is clamped to + `backoff_max`. This is intentional (bounded, never indefinite), but interactive callers who + cannot tolerate multi-minute blocking should lower `rate_limit_max_retries` / + `rate_limit_backoff_max`, or disable retry and handle 429s themselves. + +## Complexity Tracking + +> No constitution violations; table intentionally empty. diff --git a/dev/specs/ihs-249-sdk-429-retry/quickstart.md b/dev/specs/ihs-249-sdk-429-retry/quickstart.md new file mode 100644 index 000000000..c9611ea7b --- /dev/null +++ b/dev/specs/ihs-249-sdk-429-retry/quickstart.md @@ -0,0 +1,70 @@ +# Quickstart / Validation Guide: SDK 429 retry with backoff + +Validates the feature end-to-end. Assumes the repo dev setup. + +## Prerequisites + +```bash +uv sync --all-groups --all-extras +``` + +## Run the unit + client tests + +```bash +uv run pytest tests/unit/test_rate_limit.py tests/unit/sdk/test_rate_limit_retry.py -v +``` + +## Validation scenarios (each maps to a Success Criterion) + +Client-level tests use a mocked `requester` / `sync_requester` (via `Config`) or a mocked +httpx transport that returns a scripted sequence of responses. + +1. **SC-001 — transparent retry-and-succeed**: script `[429, 200]`. Issue any client call. + Expect the 200 payload returned and no exception. Assert the transport was called twice. + +2. **SC-002 — honour `Retry-After`**: script a `429` carrying `Retry-After: 2` then `200`. + Patch the driver's sleep to record its argument. Expect the recorded wait ≈ 2s (clamped to + `rate_limit_backoff_max`). Repeat with an HTTP-date form and with `Retry-After: 0` (≈0s). + +3. **SC-003 — jittered exponential backoff**: script persistent `429`. Record the sleep + arguments. Assert each recorded wait ≤ `rate_limit_backoff_max`, the deterministic ceiling + (`compute_backoff`) grows exponentially, and two separate runs produce different sequences. + +4. **SC-004 — clean give-up**: script persistent `429` with `rate_limit_max_retries=5`. + Expect exactly 6 transport calls and exactly one `RateLimitError`; assert `err.attempts == 6`, + `err.url` is set, and `err.__cause__` is an `httpx.HTTPStatusError`. + +5. **SC-005 — async/sync parity**: run scenarios 1–4 parametrized over `InfrahubClient` and + `InfrahubClientSync`; assert identical attempt counts, waits (within jitter tolerance), and + error type. + +6. **SC-006 — disabled path**: set `rate_limit_retry_enabled=False`, script `[429]`. Expect the + underlying HTTP error to surface immediately (no `RateLimitError`, no wait, single transport call). + +7. **FR-006 — all request paths**: parametrize scenario 1 across a regular query/mutation + (`_request`), a multipart upload (`_request_multipart`), and streaming initiation + (`_get_streaming`); assert retry occurs on each. + +7a. **E2/X1 — multipart body survives retry**: script a multipart upload that returns `429` then + `200`, using a file payload with non-empty content. Capture the request body the transport + receives on each attempt and assert the **second attempt carries the full body** (equal to the + first), proving the payload was rewound / re-materialized rather than sent as a consumed stream. + +8. **FR-007 — logging**: with `caplog`, assert a `WARNING` record per retry containing the URL, + attempt number, and delay. + +## Handler unit checks (pure, no I/O) + +```bash +uv run pytest tests/unit/test_rate_limit.py -v +``` + +Covers: `compute_backoff` growth + clamp; `jittered_delay` range + variance; `parse_retry_after` +for delta-seconds, HTTP-date, past date (→0), and malformed (→None); `next_delay` clamping and +`Retry-After`-vs-computed selection; `should_retry` budget (`max_retries + 1` total). + +## Manual smoke (optional) + +Point a real client at a server that returns 429 (or a local stub), issue a bulk operation, and +observe in logs that the SDK backs off and either succeeds or raises `RateLimitError` after the +configured retries. diff --git a/dev/specs/ihs-249-sdk-429-retry/research.md b/dev/specs/ihs-249-sdk-429-retry/research.md new file mode 100644 index 000000000..014175024 --- /dev/null +++ b/dev/specs/ihs-249-sdk-429-retry/research.md @@ -0,0 +1,166 @@ +# Research: SDK retry with backoff on HTTP 429 responses + +## R1 — Where do 429s actually surface? (chokepoint audit) + +**Decision**: Apply the retry driver at three send sites per client, not only `_request`. + +**Findings** (from `infrahub_sdk/client.py`): + +- `_request` (async ~L1486, sync equivalent) calls `self._request_method` (default + `_default_request_method`, overridable via `Config.requester`) and returns the raw + `httpx.Response`. `_post`, `_get`, `login`, and `refresh_login` all funnel through + `_request` — so **queries, mutations, and auth are covered by wrapping `_request`**. +- `_request_multipart` (async ~L1383) builds its own `httpx.AsyncClient` and calls + `client.post(...)` directly — it **bypasses `_request`**. Must be wrapped separately + to satisfy FR-006 (multipart uploads). +- `_get_streaming` (async ~L1455) is an `@asynccontextmanager` that opens `client.stream(...)` + directly — it **bypasses `_request`**. Retry must wrap the *initiation* of the stream. + +**Sync client is symmetric.** `InfrahubClientSync` (class @ L2053) mirrors the async client +exactly: `_request` (L3583) is the funnel for `_get`/`_post`/`login`/`refresh_login`, while +`_request_multipart` (L2343 send) and `_get_streaming` (L3545 send) each build their own +`httpx.Client` and bypass `_request`. So the same three-send-site treatment applies to **both** +clients — six send sites total. + +**Exhaustive send-site audit.** Enumerating every direct httpx send in `client.py` +(`client.request` / `client.post` / `client.stream`) yields exactly six call sites — three per +client, listed above. The many `response.raise_for_status()` lines are response *consumers* that +run on responses already obtained via those send sites, not new send paths. There is therefore no +fourth path to cover on either client. + +**Rationale**: The PRD assumed a single `_request` chokepoint; the code shows two additional +send paths per client. Covering all three (on each of the async and sync clients) is required for +FR-006 and FR-008. A shared retry driver — one async variant (`asyncio.sleep`) and one sync +variant (`time.sleep`), both consuming the same pure `RateLimitRetryHandler` — wraps a "perform +one send, return the response" callable so all six call sites behave identically. + +**Alternatives considered**: + +- *Refactor multipart/streaming to route through `_request`*: larger blast radius, changes + more code paths, risks regressions in streaming/upload behaviour. Rejected in favour of + wrapping each send site with the same driver. +- *Retry only `_request`*: simplest but violates FR-006 (multipart + streaming uncovered). Rejected. + +## R2 — Detecting a 429 without disturbing existing error flow + +**Decision**: Inspect `response.status_code == 429` on the returned response inside the driver; +do not rely on `raise_for_status()`. + +**Rationale**: `_request`/`_request_multipart` return the raw response; callers call +`raise_for_status()` afterwards. Inspecting the status code directly lets the driver decide to +retry before any exception is raised, and preserves the existing behaviour for every non-429 +response (returned untouched). On exhaustion the driver synthesizes the terminal error by +calling `response.raise_for_status()` (which raises `httpx.HTTPStatusError`) and chains it as +the `__cause__` of `RateLimitError`. + +**Alternatives considered**: Catching `httpx.HTTPStatusError` around callers — rejected because +`_request` doesn't raise it and the catch sites are scattered. + +## R3 — Backoff algorithm (FR-002, SC-003) + +**Decision**: `computed = min(backoff_max, backoff_base * 2**attempt)`, then apply full jitter: +`delay = random.uniform(0, computed)`. `attempt` is 0-indexed per request. + +**Rationale**: "Full jitter" (AWS Architecture Blog, *Exponential Backoff And Jitter*) minimises +thundering-herd re-saturation better than equal/decorrelated jitter for this use case, and +trivially satisfies SC-003 (two instances differ). The base×2^attempt term grows exponentially +until clamped to `backoff_max`. + +**Note on SC-003 "successive waits grow exponentially"**: because full jitter samples in +`[0, computed]`, an individual sampled sequence is not monotonic. The *ceiling* (`computed`, the +upper bound) grows exponentially and is clamped; the handler exposes both the clamped ceiling and +the jittered delay so tests can assert the ceiling growth deterministically and assert jitter +presence separately. See data-model.md. + +**Alternatives considered**: Equal jitter (`computed/2 + uniform(0, computed/2)`) — also valid; +full jitter chosen for maximum de-correlation. `random.random()`-based — equivalent, `uniform` +is clearer. + +## R4 — Parsing `Retry-After` (FR-003, FR-004, edge cases) + +**Decision**: Support both RFC 7231 forms; on any parse failure fall back to computed backoff. + +- **delta-seconds**: `int(value)` → seconds. +- **HTTP-date**: `email.utils.parsedate_to_datetime(value)` (stdlib), then + `(parsed - now).total_seconds()`, floored at `0` (past dates → 0, never negative). +- **Malformed / unparseable** (non-numeric, bad date, empty): return `None` → driver uses + computed backoff (FR-004). +- Result is clamped to `backoff_max` in all cases (FR-003). + +**Rationale**: `email.utils.parsedate_to_datetime` is stdlib and handles RFC-compliant HTTP-dates +(it returns timezone-aware datetimes for GMT). Flooring at 0 handles the past-date edge case. + +**"now" injection**: to keep the handler pure/testable, the HTTP-date branch takes an injectable +`now` callable (defaults to `datetime.now(timezone.utc)`); tests pass a fixed `now`. + +**Alternatives considered**: `dateutil` — rejected (new dependency). Hand-rolled date parsing — +rejected (error-prone). + +## R5 — Config surface (FR-009) + +**Decision**: Add four fields to `ConfigBase` (so both `Config` and subclasses inherit) using +pydantic `Field` with descriptions, matching the existing `retry_delay` / `retry_on_failure` style: + +- `rate_limit_retry_enabled: bool = True` +- `rate_limit_max_retries: int = 5` +- `rate_limit_backoff_base: float = 0.5` +- `rate_limit_backoff_max: float = 60.0` + +**Rationale**: `ConfigBase` (`infrahub_sdk/config.py:38`) already holds `retry_delay`, +`retry_on_failure`, `max_retry_duration`, `timeout` — the rate-limit knobs belong alongside them. +They are independent of the existing `retry_on_failure` mechanism (which is not modified — out of scope). + +**Alternatives considered**: A nested `rate_limit` sub-model — rejected as heavier than the flat +style already used; four flat fields match repo convention and keep env-var mapping simple. + +## R6 — `RateLimitError` shape (FR-005) + +**Decision**: `class RateLimitError(Error)` with +`__init__(self, url: str, attempts: int, retry_after: float | None = None, message: str | None = None)`. +Stores `url`, `attempts`, `retry_after`; builds a default message if none given. Raised with +`raise RateLimitError(...) from http_status_error` so `__cause__` is the underlying +`httpx.HTTPStatusError` (open-question resolution). + +**Rationale**: Mirrors existing `Error` subclasses in `exceptions.py` (e.g. `JsonDecodeError` +carries `url`). Chaining via `from` preserves the raw response for callers (SC / assumptions). + +## R7 — Attempt accounting (FR-001, P3/SC-004) + +**Decision**: `max_retries` counts *retries*, so total sends = `max_retries + 1` (one initial + +N retries). The handler decides "exhausted" when the number of retries already performed equals +`max_retries`. + +**Rationale**: Matches the PRD's P3 acceptance ("exactly `max_retries + 1` attempts") and SC-004. + +## R8 — Logging (FR-007) + +**Decision**: The driver logs one record per retry via the SDK's existing module logger, +including URL, attempt number (1-based), and the computed/honoured delay. Level: `WARNING` +(rate-limiting is an operational condition worth surfacing) — consistent with observable-but- +non-fatal events. + +**Rationale**: FR-007 requires each retry be observable. Using the existing logger keeps it +configurable by the host application. + +**Log-content constraint (critique E4)**: retry log records MUST contain only the request URL, +the attempt number, and the applied delay — never request headers or payload. The login and +token-refresh paths carry `Authorization: Bearer …` headers and username/password payloads, so +broadening the log content would leak credentials. Tests assert the presence of URL/attempt/delay; +implementation must not add headers/body to the record. + +## R9 — Build vs buy (critique P3) + +**Decision**: Implement a small custom handler rather than adopt a retry library. + +**Rationale**: + +- **httpx transport-level `retries`** (`httpx.HTTPTransport(retries=N)`) retries only connection + establishment failures, not HTTP status codes — it cannot see a 429, so it cannot satisfy FR-001/003. +- **`tenacity`** would be a new runtime dependency, which is out of scope ("no new dependency"), + and would still need custom predicates for 429 detection, `Retry-After` parsing, and the + `RateLimitError` contract — most of the logic we'd write anyway. +- The required logic (parse `Retry-After`, jittered/clamped backoff, attempt budget) is small, + pure, and fully unit-testable with stdlib only. + +**Alternatives considered**: `tenacity`, `backoff`, httpx transport retries — all rejected for the +reasons above. diff --git a/dev/specs/ihs-249-sdk-429-retry/spec.md b/dev/specs/ihs-249-sdk-429-retry/spec.md new file mode 100644 index 000000000..1835337a1 --- /dev/null +++ b/dev/specs/ihs-249-sdk-429-retry/spec.md @@ -0,0 +1,131 @@ +# Feature Specification: SDK retry with backoff on HTTP 429 responses + +**Feature Branch**: `dga/feat-409-retry-ivj0i` + +**Created**: 2026-07-07 + +**Status**: Draft + +**Input**: Jira IHS-249 — "SDK retry with backoff on HTTP 429 responses"; GitHub issue opsmill/infrahub-sdk-python#1124 + +## User Scenarios & Testing *(mandatory)* + +### User Story 1 - Transparent retry-and-succeed (Priority: P1) + +A caller makes a request through the SDK. The server returns HTTP 429 (Too Many Requests), with or without a `Retry-After` header. The SDK waits and retries automatically; the request then succeeds and the caller receives the result with no error and no retry code of their own. + +**Why this priority**: This is the core value of the feature — transient rate-limiting stops failing scripts and callers no longer need to hand-write retry loops. Without it the feature delivers nothing; with it alone the SDK is already meaningfully more resilient. + +**Independent Test**: Point a client at a transport that returns one 429 then a 200, issue any request, and confirm the caller receives the 200 result with no exception raised. Fully testable in isolation and delivers immediate value. + +**Acceptance Scenarios**: + +1. **Given** a client whose next request will receive one 429 followed by a 200, **When** the caller issues the request, **Then** the SDK returns the 200 result transparently and the caller observes no error. +2. **Given** rate-limit retry is enabled (the default), **When** a 429 is received, **Then** the SDK waits before re-issuing the same request rather than surfacing the 429 immediately. + +--- + +### User Story 2 - Respect `Retry-After` (Priority: P2) + +When the server returns a 429 carrying a `Retry-After` header, the SDK waits the server-specified duration before retrying, parsing both the delta-seconds form (`Retry-After: 5`) and the HTTP-date form (`Retry-After: Wed, 21 Oct 2026 07:28:00 GMT`). The wait is clamped to the configured maximum. + +**Why this priority**: Honouring `Retry-After` is what lets a load-shedding server control exactly when background SDK traffic returns. It builds directly on P1 and is the cooperative-backoff contract the server-side prioritisation work (INFP-636) depends on. + +**Independent Test**: Return a 429 with `Retry-After: N` (once in delta-seconds form, once in HTTP-date form) followed by a 200, and confirm the observed wait before the retry is approximately N seconds (or the configured maximum when N exceeds it). + +**Acceptance Scenarios**: + +1. **Given** a 429 carrying `Retry-After: N` in delta-seconds form, **When** the SDK retries, **Then** the wait before the next attempt is approximately N seconds (clamped to the configured maximum if N exceeds it). +2. **Given** a 429 carrying `Retry-After` as an HTTP-date, **When** the SDK retries, **Then** the wait before the next attempt is approximately the interval between now and that date (clamped to the maximum, and never negative). +3. **Given** a 429 whose `Retry-After` header is malformed or unparseable, **When** the SDK retries, **Then** the retry still happens using the computed exponential backoff and no error is raised over the bad header. + +--- + +### User Story 3 - Give up cleanly on sustained rate-limiting (Priority: P3) + +The server returns 429 on every attempt. After the configured maximum number of retries the SDK stops trying and raises a dedicated `RateLimitError`, having logged each attempt. The error carries enough context (the URL, the number of attempts made, and the last `Retry-After` seen) for the caller to react. + +**Why this priority**: A hard cap prevents a persistently overloaded server from hanging a caller indefinitely and gives callers a clear, catchable failure distinct from other HTTP errors. It depends on P1's retry loop already existing. + +**Independent Test**: Point a client at a transport that always returns 429, issue a request, and confirm exactly `max_retries + 1` attempts are made and exactly one `RateLimitError` is raised carrying the URL, attempt count, and last `Retry-After`. + +**Acceptance Scenarios**: + +1. **Given** a server that always returns 429, **When** the caller issues a request, **Then** exactly `max_retries + 1` total attempts are made and a single `RateLimitError` is raised. +2. **Given** retries have been exhausted, **When** the `RateLimitError` is raised, **Then** it exposes the request URL, the number of attempts made, and the last `Retry-After` value observed. +3. **Given** each retry occurs, **When** the SDK waits, **Then** it emits a log record identifying the URL, the attempt number, and the delay applied. + +--- + +### User Story 4 - Tune or disable the behaviour (Priority: P3) + +A developer whose needs differ from the defaults adjusts the retry behaviour — or turns it off entirely — through configuration, without changing any call sites. + +**Why this priority**: Escape hatches matter for callers who already have their own retry strategy or who need deterministic failure. It is additive and does not block the core journeys. + +**Independent Test**: Set the disable flag in configuration, return a single 429, and confirm the SDK raises immediately without retrying (matching the pre-feature behaviour path). Separately, lower the maximum-retries value and confirm the attempt count follows. + +**Acceptance Scenarios**: + +1. **Given** rate-limit retry is disabled via configuration, **When** a 429 is received, **Then** the SDK surfaces the error immediately with no wait and no retry. +2. **Given** the maximum retries and backoff bounds are changed via configuration, **When** a persistent 429 occurs, **Then** the observed attempt count and waits follow the configured values. +3. **Given** identical configuration, **When** the same 429 sequence is driven through the asynchronous client and the synchronous client, **Then** both produce identical observable behaviour (attempt counts, waits within jitter tolerance, and the same error type). + +--- + +### Edge Cases + +- **`Retry-After` as a past HTTP-date**: treated as a zero / minimal wait, never a negative delay. +- **`Retry-After` malformed or unparseable**: ignored; the SDK falls back to computed exponential backoff and still retries. +- **`Retry-After` larger than the configured maximum wait**: clamped down to the configured maximum. +- **429 on a mutating request (create/update/upload)**: safe to retry, because a 429 is a pre-processing rejection with no partial write on the server. +- **Many concurrent clients hitting the same 429**: jitter in the computed backoff prevents them retrying in lockstep and re-saturating the server (thundering herd). +- **Retry disabled via configuration**: a 429 raises immediately, preserving the existing behaviour path. +- **429 exhaustion**: the caller sees a `RateLimitError` rather than the raw transport error, but can still inspect the underlying HTTP error through the raised exception's cause. + +## Requirements *(mandatory)* + +### Functional Requirements + +- **FR-001**: The SDK MUST retry a request that receives HTTP 429, up to a configurable maximum number of attempts. +- **FR-002**: Between retries the SDK MUST wait using exponential backoff with random jitter, and MUST clamp each computed wait to a configurable maximum. Successive computed waits MUST grow until the ceiling is reached, and two client instances MUST NOT produce identical wait sequences (jitter must be present). +- **FR-003**: When a 429 response includes a `Retry-After` header, the SDK MUST honour it in place of the computed backoff, parsing both the delta-seconds form and the HTTP-date form, and MUST clamp the resulting wait to the configured maximum. +- **FR-004**: A `Retry-After` header that is malformed or unparseable MUST NOT crash the client or cause the retry to be skipped; the SDK MUST fall back to computed exponential backoff. +- **FR-005**: When retries are exhausted, the SDK MUST raise a dedicated, catchable rate-limit error that is distinct from other HTTP errors and carries the request URL, the number of attempts made, and the last `Retry-After` value observed. The error MUST preserve the underlying transport HTTP error as its cause so callers can inspect the raw response. +- **FR-006**: Retry behaviour MUST apply to every request path where a 429 can occur — including queries, mutations, multipart uploads, streaming initiation, and authentication requests. +- **FR-007**: The SDK MUST log each retry, including the request URL, the attempt number, and the delay applied. +- **FR-008**: Retry behaviour MUST be identical between the asynchronous client and the synchronous client. +- **FR-009**: Users MUST be able to tune the retry behaviour — and to disable it entirely — through configuration. When disabled, a 429 MUST surface immediately without any retry. + +### Key Entities *(include if feature involves data)* + +- **Rate-limit retry configuration**: the set of tunable values that govern the behaviour — whether retry is enabled, the maximum number of retries, the base backoff interval, and the maximum backoff interval. Ships with sensible defaults (enabled, five retries, half-second base, sixty-second ceiling) and is exposed through the SDK's existing configuration surface. +- **Rate-limit retry decision logic**: pure logic (no input/output) that parses `Retry-After`, computes jittered exponential backoff, clamps waits to the maximum, and decides whether to continue retrying or declare exhaustion. Consumed identically by both clients. +- **Rate-limit error**: the dedicated exception raised on exhaustion. A subtype of the SDK's base error, carrying the request URL, attempts made, and last `Retry-After` seen, with the underlying transport error preserved as its cause. + +## Success Criteria *(mandatory)* + +### Measurable Outcomes + +- **SC-001**: A request that receives a 429 then a success returns the successful result transparently, with no error surfaced and no caller-side retry code. +- **SC-002**: With `Retry-After: N` present, the wait before the next attempt is within jitter tolerance of N seconds, and approximately zero when the header indicates zero or a past date. +- **SC-003**: Without `Retry-After`, successive waits grow exponentially, never exceed the configured maximum, and differ between two independent client instances (demonstrating jitter). +- **SC-004**: After the configured maximum consecutive 429s, exactly one rate-limit error is raised and no further requests are attempted (total attempts equal maximum retries plus one). +- **SC-005**: The observable behaviour — attempt counts, waits within jitter tolerance, results, and error type — is identical across the asynchronous and synchronous clients. +- **SC-006**: With retry disabled through configuration, a single 429 surfaces immediately with no wait and no additional attempt. + +## Assumptions + +- A 429 is a pre-processing rejection by the server, so retrying any request method — including mutations and uploads — is safe and cannot cause a partial write. +- The server communicates recovery time via a standard `Retry-After` header when it chooses to; its absence is normal and handled by computed backoff. +- All 429-returning traffic flows through the clients' shared request chokepoint, so the retry loop can be applied in one place and cover every request path. +- The rate-limit error preserving the underlying transport error as its cause is the desired resolution of the PRD's open question, chosen because it is low cost and preserves the caller's ability to inspect the raw response. +- Default configuration values (enabled, five retries, half-second base backoff, sixty-second maximum backoff) are appropriate for typical background workloads and can be overridden per caller. +- The existing connectivity-level retry mechanism (`retry_on_failure`) is independent and remains unchanged; this feature does not modify or unify it. + +## Out of Scope + +- Retrying HTTP status codes other than 429 (for example 503). +- Server-side rate limiting, origin/priority signalling, or dedicated API capacity — those are the server-side halves of INFP-636 and INFP-635, tracked separately. +- Changing or unifying the existing connectivity `retry_on_failure` mechanism. +- Any new CLI commands or configuration surface beyond the additive rate-limit settings; `infrahubctl` inherits the behaviour transparently. diff --git a/dev/specs/ihs-249-sdk-429-retry/tasks.md b/dev/specs/ihs-249-sdk-429-retry/tasks.md new file mode 100644 index 000000000..e3bd83366 --- /dev/null +++ b/dev/specs/ihs-249-sdk-429-retry/tasks.md @@ -0,0 +1,155 @@ +--- + +description: "Task list for SDK retry with backoff on HTTP 429 responses (IHS-249)" +--- + +# Tasks: SDK retry with backoff on HTTP 429 responses + +**Input**: Design documents from `specs/ihs-249-sdk-429-retry/` + +**Prerequisites**: plan.md, spec.md, research.md, data-model.md, contracts/, quickstart.md (all present; critique applied) + +**Tests**: INCLUDED — the spec's Testing Decisions and the feature request explicitly require unit tests for the pure handler and client-level tests parametrized across the async and sync clients. + +**Organization**: Tasks are grouped by user story. Foundational phase builds the shared retry machinery (handler, error, config, drivers wired into all three send sites on both clients); the multipart body re-read fix (critique E2/X1 — Must-Address) lives there because every path flows through it. + +## Format: `[ID] [P?] [Story] Description` + +- **[P]**: Can run in parallel (different files, no dependencies) +- **[Story]**: Which user story this task belongs to (US1–US4) + +## Path Conventions + +Single-project library: source under `infrahub_sdk/`, tests under `tests/unit/`. + +--- + +## Phase 1: Setup (Shared Infrastructure) + +**Purpose**: Create the new module and test files the feature will fill in. + +- [X] T001 [P] Create new module `infrahub_sdk/rate_limit.py` with imports (`from __future__ import annotations`, `random`, `datetime`/`timezone`, `email.utils.parsedate_to_datetime`) and an empty `RateLimitRetryHandler` class stub. +- [X] T002 [P] Create test files `tests/unit/test_rate_limit.py` (handler unit tests) and `tests/unit/sdk/test_rate_limit_retry.py` (client-level tests) with module docstrings and pytest imports. + +--- + +## Phase 2: Foundational (Blocking Prerequisites) + +**Purpose**: Build the shared retry machinery every user story depends on. Covers FR-001, FR-002, FR-003, FR-004, FR-005, FR-006, FR-007, FR-008, and the E2/X1 multipart Must-Address. + +**⚠️ CRITICAL**: No user story work can begin until this phase is complete. + +- [X] T003 [P] Add four fields to `ConfigBase` in `infrahub_sdk/config.py` (alongside `retry_on_failure`/`retry_delay`): `rate_limit_retry_enabled: bool = True`, `rate_limit_max_retries: int = Field(default=5, ge=0)`, `rate_limit_backoff_base: float = Field(default=0.5, gt=0)`, `rate_limit_backoff_max: float = Field(default=60.0, gt=0)`, each with a `description=` per `contracts/config.md`. (FR-009) +- [X] T004 [P] Add `RateLimitError(Error)` to `infrahub_sdk/exceptions.py` with `__init__(self, url, attempts, retry_after=None, message=None)` storing `url`/`attempts`/`retry_after` and building a default message, per `contracts/rate_limit_error.md`. (FR-005) +- [X] T005 Implement `RateLimitRetryHandler` in `infrahub_sdk/rate_limit.py`: `__init__(max_retries, backoff_base, backoff_max)`, `parse_retry_after(header, *, now=None)` (delta-seconds via `int`; HTTP-date via `parsedate_to_datetime` floored at 0; malformed→`None`), `compute_backoff(attempt)` = `min(backoff_max, backoff_base * 2**attempt)`, `jittered_delay(ceiling)` = `random.uniform(0, ceiling)`, `next_delay(attempt, retry_after_header=None, *, now=None)` (honour parsed Retry-After clamped to max, else jittered backoff clamped to max), `should_retry(attempts_made)` = `attempts_made <= max_retries`. Per `contracts/rate_limit_retry_handler.md`. (FR-002, FR-003, FR-004) +- [X] T006 [P] Write handler unit tests in `tests/unit/test_rate_limit.py`: `compute_backoff` growth + clamp to `backoff_max`; `jittered_delay(c)` ∈ `[0, c]` and a sample of draws varies; `parse_retry_after` for delta-seconds, HTTP-date (fixed injected `now`), past date → `0.0`, malformed/empty → `None`; `next_delay` clamping and Retry-After-vs-computed selection; `should_retry` yields exactly `max_retries + 1` total sends. (Depends on T005 signatures; write to fail first.) +- [X] T007 Implement the async retry driver `_send_with_rate_limit_retry(self, send, url)` on `InfrahubClient` in `infrahub_sdk/client.py`: if `not config.rate_limit_retry_enabled` return `await send()`; else loop calling `send()`, count attempts, return on non-429, on 429 either sleep `await asyncio.sleep(handler.next_delay(...))` and log a `WARNING` (url, attempt, delay), or when `not handler.should_retry(...)` build `httpx.HTTPStatusError` via `response.raise_for_status()` and `raise RateLimitError(url, attempts, last_retry_after) from exc`. Wire it into `_request`. (FR-001, FR-005, FR-007, FR-009; depends on T003–T005) +- [X] T008 Implement the sync retry driver `_send_with_rate_limit_retry` on `InfrahubClientSync` in `infrahub_sdk/client.py` with identical logic using `time.sleep`, wired into the sync `_request`. Keep logic byte-for-byte parallel to the async variant (FR-008). (Depends on T003–T005) +- [X] T009 Wire the retry driver into `_request_multipart` on BOTH clients in `infrahub_sdk/client.py` — async `InfrahubClient._request_multipart` (L1383) and sync `InfrahubClientSync._request_multipart` (L2331) — AND implement the E2/X1 Must-Address fix on each: before each attempt, rewind every file object in the `files` payload (`seek(0)`) or materialize the multipart body to bytes once and re-send those bytes, so a retried upload carries the full body. (FR-006, FR-008 + critique E2/X1; depends on T007, T008) +- [X] T010 Wire the retry driver into `_get_streaming` on BOTH clients in `infrahub_sdk/client.py` — async `InfrahubClient._get_streaming` (L1455) and sync `InfrahubClientSync._get_streaming` (L3524) — so a 429 on stream initiation is retried before any body is consumed; the driver wraps opening the stream and reading the response status. (FR-006, FR-008; depends on T007, T008) + +**Checkpoint**: Retry machinery is complete and applied on all three send sites of both clients. User story validation can now proceed. + +--- + +## Phase 3: User Story 1 - Transparent retry-and-succeed (Priority: P1) 🎯 MVP + +**Goal**: A 429 followed by a 200 returns the 200 result transparently, no error, no caller retry code. + +**Independent Test**: Mock a transport returning `[429, 200]`; issue a request; assert the 200 payload is returned, no exception raised, and the transport was called twice. + +- [X] T011 [P] [US1] Client-level test in `tests/unit/sdk/test_rate_limit_retry.py`: script `[429, 200]` via a mocked `requester`/`sync_requester` (or mocked transport), parametrized across `InfrahubClient` and `InfrahubClientSync`; assert result returned transparently, no exception, exactly two sends. Patch the driver sleep to avoid real waits. (SC-001) +- [X] T012 [US1] Confirm the `_request` path (used by `_get`/`_post`/`login`/`refresh_login`) returns non-429 responses untouched and retries a 429 transparently; adjust T007/T008 if the test reveals a gap. (SC-001) + +**Checkpoint**: MVP — the SDK transparently rides through a transient 429 on both clients. + +--- + +## Phase 4: User Story 2 - Respect `Retry-After` (Priority: P2) + +**Goal**: The SDK waits the server-specified `Retry-After` duration (delta-seconds and HTTP-date), clamped to max, before retrying. + +**Independent Test**: Script `429` with `Retry-After` then `200`; capture the driver's sleep argument; assert it ≈ header value (and ≈0 for a zero/past value, clamped when larger than max). + +- [X] T013 [P] [US2] Client-level tests in `tests/unit/sdk/test_rate_limit_retry.py` (parametrized async+sync): (a) `Retry-After: N` delta-seconds → wait ≈ N; (b) HTTP-date form → wait ≈ interval; (c) `Retry-After: 0` and past date → wait ≈ 0; (d) malformed header → falls back to computed backoff and still retries; (e) `Retry-After` > `rate_limit_backoff_max` → clamped to max. Patch/record the sleep argument. (SC-002, FR-003, FR-004) + +**Checkpoint**: Server-directed backoff honoured on both clients. + +--- + +## Phase 5: User Story 3 - Give up cleanly on sustained rate-limiting (Priority: P3) + +**Goal**: Persistent 429 → after `rate_limit_max_retries` retries, raise one `RateLimitError` (with url/attempts/retry_after and chained `__cause__`), having logged each retry. + +**Independent Test**: Script persistent `429` with `max_retries=5`; assert exactly 6 sends, one `RateLimitError`, its attributes, and one WARNING log per retry. + +- [X] T014 [P] [US3] Client-level tests in `tests/unit/sdk/test_rate_limit_retry.py` (parametrized async+sync): persistent `429` → exactly `max_retries + 1` sends; exactly one `RateLimitError` raised; assert `err.url`, `err.attempts == max_retries + 1`, `err.retry_after`, and `isinstance(err.__cause__, httpx.HTTPStatusError)`; with `caplog`, assert one `WARNING` per retry containing url, attempt number, and delay. (SC-004, FR-005, FR-007) +- [X] T015 [US3] Verify the driver (T007/T008) synthesizes the terminal `httpx.HTTPStatusError` from the final 429 response and chains it as `RateLimitError.__cause__`, and tracks `last_retry_after`; refine if T014 fails. (FR-005) + +**Checkpoint**: Clean, catchable, observable exhaustion on both clients. + +--- + +## Phase 6: User Story 4 - Tune or disable the behaviour (Priority: P3) + +**Goal**: Retry is tunable and fully disableable via `Config`, with identical behaviour across async and sync. + +**Independent Test**: With `rate_limit_retry_enabled=False`, a single 429 raises immediately (no wait, one send); with altered `max_retries`/backoff, attempt counts and waits follow config. + +- [X] T016 [P] [US4] Client-level tests in `tests/unit/sdk/test_rate_limit_retry.py` (parametrized async+sync): (a) `rate_limit_retry_enabled=False` → a 429 surfaces the underlying HTTP error immediately, no `RateLimitError`, no wait, one send (SC-006, FR-009); (b) lowered `rate_limit_max_retries` → observed attempt count follows; (c) explicit async/sync parity assertion — same 429 sequence yields identical attempt counts, waits within jitter tolerance, and same error type (SC-005, FR-008). + +**Checkpoint**: All four user stories independently functional and validated on both clients. + +--- + +## Phase 7: Polish & Cross-Cutting Concerns + +**Purpose**: FR-006 all-paths coverage, the E2/X1 regression guard, changelog, and repo gates. + +- [X] T017 [P] FR-006 all-paths test in `tests/unit/sdk/test_rate_limit_retry.py`: parametrize a `429→200` retry across a regular request, a multipart upload (`_request_multipart`), and streaming initiation (`_get_streaming`), on both clients; assert retry occurs on each. (FR-006) +- [X] T018 [P] E2/X1 regression test in `tests/unit/sdk/test_rate_limit_retry.py`: a multipart upload returning `429` then `200` with non-empty file content; capture the body the transport receives per attempt and assert the second attempt carries the full body equal to the first (proves payload rewind/re-materialize). (Critique E2/X1) +- [X] T019 [P] Add towncrier changelog fragments in `changelog/`: `1124.added.md` (transparent 429 retry with jittered backoff, `Retry-After` support, four `rate_limit_*` Config fields, new `RateLimitError`) and `1124.changed.md` (a persistent 429 now raises `RateLimitError` after retries exhaust instead of `httpx.HTTPStatusError`; the raw error is available via `__cause__`). +- [X] T020 Run `uv run invoke docs-generate` (Config gained public fields) and confirm generated SDK docs update; do not hand-edit generated files. +- [X] T021 Run `uv run invoke format lint-code` and `uv run pytest tests/unit/test_rate_limit.py tests/unit/sdk/test_rate_limit_retry.py` — all green (quickstart.md validation). + +--- + +## Dependencies & Execution Order + +### Phase Dependencies + +- **Setup (Phase 1)**: No dependencies — start immediately. +- **Foundational (Phase 2)**: Depends on Setup. BLOCKS all user stories. Internal order: T003/T004 [P] → T005 → T006 [P] / T007 / T008 → T009 / T010. +- **User Stories (Phases 3–6)**: All depend on Foundational completion. Because the machinery is shared, the stories are validation-led and can run in parallel once Phase 2 is done; recommended order P1 → P2 → P3 → P3. +- **Polish (Phase 7)**: Depends on Foundational (T017/T018) and all stories for T021. + +### Within Each User Story + +- Tests are written to fail first, then the foundational implementation is confirmed/adjusted to make them pass. + +### Parallel Opportunities + +- T001, T002 in parallel. +- T003, T004 in parallel; T006 parallel with T007/T008 once T005 lands. +- Story test tasks T011, T013, T014, T016 touch the same test file — treat as sequential edits (do NOT run in parallel to avoid conflicts) unless split into separate test functions by different agents; T017/T018/T019 are [P] across different files (T019 is changelog). + +--- + +## Implementation Strategy + +### MVP First (User Story 1) + +1. Phase 1 Setup → 2. Phase 2 Foundational (critical) → 3. Phase 3 US1 → validate `[429, 200]` transparent success on both clients → demo. + +### Incremental Delivery + +Foundation → US1 (MVP) → US2 (Retry-After) → US3 (clean give-up) → US4 (tune/disable + parity) → Polish (FR-006 coverage, E2 regression, changelog, gates). Each story is independently testable against the shared machinery. + +--- + +## Notes + +- [P] = different files, no dependencies. The single client test file makes most story test tasks sequential edits. +- The async and sync drivers (T007/T008) must stay logically identical (FR-008); review them together. +- Do not modify generated code (`protocols.py`). Run `docs-generate` for the new Config fields (T020). +- The multipart re-read fix (T009) is the critique's Must-Address — do not skip its regression test (T018). diff --git a/dev/specs/ihs-259-sdk-x-priority-header/alignment-check.md b/dev/specs/ihs-259-sdk-x-priority-header/alignment-check.md new file mode 100644 index 000000000..2c3a7e621 --- /dev/null +++ b/dev/specs/ihs-259-sdk-x-priority-header/alignment-check.md @@ -0,0 +1,32 @@ +# Spec / Ask Alignment Check: SDK `X-Priority` Request Header + +**Date**: 2026-07-10 +**Feature dir**: `specs/ihs-259-sdk-x-priority-header/` + +## 1. Source + +- **Source PRD**: Jira **IHS-259** — "feat: SDK X-Priority request header" (`https://opsmill.atlassian.net/browse/IHS-259`), fetched via the Atlassian integration. The Jira issue description *is* a full PRD (Problem Statement, Solution Overview, 7 User Stories, FR-001…008, Key Entities, Edge Cases, SC-001…005, Implementation/Testing Decisions, Out of Scope, Assumptions). +- Compared against: `spec.md` (current, post-critique). + +## 2. Verdict + +✅ **ALIGNED** + +The spec faithfully carries every PRD requirement, user story, acceptance criterion, edge case, and out-of-scope boundary. The only differences are expansions of detail and testability clarifications that preserve — and in two cases make verifiable — the PRD's stated intent. No requirement is missing, changed in meaning, dropped, softened, or contradicted. + +## 3. Findings + +| Severity | Category | PRD reference | Spec reference | Description | +|----------|----------|---------------|----------------|-------------| +| ℹ️ Info (no drift) | mapping | PRD User Stories 1–7 | spec US1–US5 | 7 PRD stories consolidated into 5. All intent preserved: PRD US1→US1, US2 (enum) folded into FR-001 + contracts, US3 (override)→US2, US4 (rides every transport)→US1/FR-003, US5 (zero change)→US3, US6 (invalid rejected)→US4, US7 (async=sync)→US5. Consolidation, not loss. | +| ℹ️ Info (expansion) | added | PRD Assumptions ("header is exactly `X-Priority`") | spec FR-009 | Spec adds FR-009 stating the header name is exactly `X-Priority` with lowercase value. This promotes a PRD assumption to a testable requirement — expansion of detail, within PRD scope. | +| ℹ️ Info (expansion) | added | PRD Edge Cases ("batch mode and raw blob transfers inherit the client default") | spec SC-006 | Spec adds SC-006 verifying batch/blob inherit the configured default. Makes an implicit PRD scope claim testable; does not add new scope (no per-request override for these, matching the PRD). Raised by the critique (P5/X1). | +| ⚠️ Minor (clarified, not softened) | changed-wording | PRD SC-002 ("emits no `X-Priority` header — asserted byte-for-byte against current behaviour") | spec SC-002 | Reworded to "no `X-Priority` emitted; no other SDK-set outgoing header changes (assert `X-Priority` absent; not a literal byte-for-byte comparison of transport-injected headers)". The requirement (no header, no behaviour change) is unchanged; only the assertion method is clarified because httpx injects its own headers, making a literal byte comparison neither stable nor meaningful. Raised by the critique (E6). | + +All FR-001…008 map 1:1 to spec FR-001…008. All SC-001, SC-003, SC-004, SC-005 map 1:1. All PRD edge cases and all five Out-of-Scope items (429/#1124, server-side/INFP-636, classification guidance, per-batch/blob knobs, anti-escalation) are present in the spec. + +## 4. Action + +**Proceed.** No remediation required. The spec is aligned with IHS-259; the two ⚠️/expansion items are testability clarifications that strengthen the spec without departing from the PRD. `tasks.md` (Phase 4) is ready for review and implementation. + +- Remediation passes used: **0**. diff --git a/dev/specs/ihs-259-sdk-x-priority-header/checklists/requirements.md b/dev/specs/ihs-259-sdk-x-priority-header/checklists/requirements.md new file mode 100644 index 000000000..12b3ea77b --- /dev/null +++ b/dev/specs/ihs-259-sdk-x-priority-header/checklists/requirements.md @@ -0,0 +1,36 @@ +# Specification Quality Checklist: SDK `X-Priority` Request Header + +**Purpose**: Validate specification completeness and quality before proceeding to planning +**Created**: 2026-07-10 +**Feature**: [spec.md](../spec.md) + +## Content Quality + +- [x] No implementation details (languages, frameworks, APIs) +- [x] Focused on user value and business needs +- [x] Written for non-technical stakeholders +- [x] All mandatory sections completed + +## Requirement Completeness + +- [x] No [NEEDS CLARIFICATION] markers remain +- [x] Requirements are testable and unambiguous +- [x] Success criteria are measurable +- [x] Success criteria are technology-agnostic (no implementation details) +- [x] All acceptance scenarios are defined +- [x] Edge cases are identified +- [x] Scope is clearly bounded +- [x] Dependencies and assumptions identified + +## Feature Readiness + +- [x] All functional requirements have clear acceptance criteria +- [x] User scenarios cover primary flows +- [x] Feature meets measurable outcomes defined in Success Criteria +- [x] No implementation details leak into specification + +## Notes + +- The spec unavoidably names the concrete wire contract (`X-Priority` header, `Priority` enum, `Config` field, method kwarg) because these ARE the requirement contract handed down from the PRD (IHS-259) and the server-side effort (INFP-636), not free implementation choices. Enum/config/kwarg names are treated as the externally observable API surface, not internal implementation detail. +- No [NEEDS CLARIFICATION] markers were needed: the source PRD is detailed and unambiguous, with resolution rules, transport coverage, and testing decisions all specified. +- All items pass. Spec is ready for `/speckit-plan`. diff --git a/dev/specs/ihs-259-sdk-x-priority-header/contracts/priority-api.md b/dev/specs/ihs-259-sdk-x-priority-header/contracts/priority-api.md new file mode 100644 index 000000000..9d4b8206b --- /dev/null +++ b/dev/specs/ihs-259-sdk-x-priority-header/contracts/priority-api.md @@ -0,0 +1,74 @@ +# API Contract: Priority public surface + +**Feature**: IHS-259 | **Scope**: SDK public Python API (async + sync). This is a public-API-signature change (governance-approved in IHS-259). + +## New public symbol: `Priority` + +```python +from infrahub_sdk.constants import Priority + +class Priority(str, enum.Enum): + HIGH = "high" + NORMAL = "normal" + LOW = "low" +``` + +- `str`-valued closed enum. `Priority("LOW") is Priority.LOW` (case-insensitive via `_missing_`). +- Unknown values raise `ValueError` (→ `pydantic.ValidationError` at config load). +- Imported from `infrahub_sdk.constants` (kept out of the top-level `infrahub_sdk` namespace so importing the enum does not pull in `Config` and the client classes). + +## Extended: `Config.priority` + +```python +class ConfigBase(BaseSettings): + ... + priority: Priority | None = Field( + default=None, + description="Default request priority emitted as the X-Priority header on every request. " + "One of high|normal|low (case-insensitive). When unset, no header is sent.", + ) +``` + +- Env var: `INFRAHUB_PRIORITY`. +- Accepts a `Priority` or a case-insensitive string; unknown → validation error at load. +- Default `None` → no client-wide default. + +## Extended method signatures (new `priority` keyword — both `InfrahubClient` and `InfrahubClientSync`) + +Each covered method gains `priority: Priority | None = None` (default `None` preserves current behaviour). The argument is keyword-friendly and additive — existing positional/keyword calls are unaffected. + +```python +# Client +def get(self, kind, ..., priority: Priority | None = None) -> ... +def all(self, kind, ..., priority: Priority | None = None) -> ... # forwards to filters + count +def filters(self, kind, ..., priority: Priority | None = None) -> ... +def count(self, kind, ..., priority: Priority | None = None) -> int +def execute_graphql(self, query, ..., priority: Priority | None = None) -> dict +def _execute_graphql_with_file(self, ..., priority: Priority | None = None) -> ... # file variant +def create_diff(self, ..., priority: Priority | None = None) -> ... +def get_diff_summary(self, ..., priority: Priority | None = None) -> ... +def get_diff_tree(self, ..., priority: Priority | None = None) -> ... + +# Node (InfrahubNode / InfrahubNodeSync) +def save(self, ..., priority: Priority | None = None) -> None +def create(self, ..., priority: Priority | None = None) -> None +def update(self, ..., priority: Priority | None = None) -> None +def delete(self, ..., priority: Priority | None = None) -> None +``` + +> **Note — `client.create` is intentionally NOT extended.** `InfrahubClient.create()` / `InfrahubClientSync.create()` only build an unsaved `InfrahubNode` in memory and issue no HTTP request, so a `priority=` kwarg there would be a no-op. Per-request priority for creating a node is carried by the node-level `save()` / `create()` (which issue the mutation). Within a node create/update, a resource-pool relationship's follow-up peer fetch inherits the same `priority`. + +### Behavioural contract per call + +- `priority=None` (default): use the client-wide default (which may itself be `None` → no header). No client state is mutated. +- `priority=Priority.X`: this request carries `X-Priority: x`, overriding the client default for this call only. The next un-annotated call reverts to the client default. +- Resolution: `resolved = per_request if per_request is not None else client_default`. + +## Explicitly NOT extended (v1) + +- `_get`, `_post`, `_get_streaming` (raw blob transfers) — inherit the client default only; no `priority` kwarg. +- Batch mode — inherits the client default only; no per-call override. + +## Backwards-compatibility guarantee + +- Adding a keyword-only-friendly parameter with a `None` default and a new optional config field is additive. Any existing caller that sets nothing sees no change in outgoing requests (FR-004 / SC-002). diff --git a/dev/specs/ihs-259-sdk-x-priority-header/contracts/x-priority-header.md b/dev/specs/ihs-259-sdk-x-priority-header/contracts/x-priority-header.md new file mode 100644 index 000000000..f9acfc0c1 --- /dev/null +++ b/dev/specs/ihs-259-sdk-x-priority-header/contracts/x-priority-header.md @@ -0,0 +1,30 @@ +# Wire Contract: `X-Priority` HTTP header + +**Feature**: IHS-259 | **Consumer**: Infrahub API server (INFP-636) + +## Header + +| Property | Value | +|----------|-------| +| Name | `X-Priority` (exact, case-insensitive on the server per HTTP header rules) | +| Values | `high`, `normal`, `low` (lowercase emitted by the SDK) | +| Cardinality | 0 or 1 per request | + +## Emission rules (SDK side) + +1. The SDK emits the header on a request **iff** the resolved priority for that request is non-`None`. +2. When emitted, the value is exactly the lowercase token of the resolved `Priority` member. +3. The header is emitted uniformly across every transport when a client-wide default is configured: GraphQL query/mutation, multipart file upload, and raw blob `_get`/`_post`. +4. When no priority is configured and none is passed per request, the header is **absent** — the outgoing request is byte-for-byte identical to the pre-feature SDK. + +## Server semantics (assumed, per INFP-636 — not implemented here) + +- The server treats the value case-insensitively. +- An **absent** header and an **unknown** value are both treated as `normal`. +- Consequently, "omit the header" and "send `normal`" are server-equivalent, which is what makes omitting-when-unconfigured a safe, non-breaking rollout. + +## Non-goals (this contract) + +- No `Retry-After` / 429 semantics (GitHub #1124). +- No server-side admission control, routing, or throttling behaviour (INFP-636). +- The SDK does not read or react to any response header related to priority. diff --git a/dev/specs/ihs-259-sdk-x-priority-header/critiques/critique-20260710-164718.md b/dev/specs/ihs-259-sdk-x-priority-header/critiques/critique-20260710-164718.md new file mode 100644 index 000000000..f33b9afbb --- /dev/null +++ b/dev/specs/ihs-259-sdk-x-priority-header/critiques/critique-20260710-164718.md @@ -0,0 +1,135 @@ +# Critique Report: SDK `X-Priority` Request Header + +**Date**: 2026-07-10 +**Feature**: [spec.md](../spec.md) +**Plan**: [plan.md](../plan.md) +**Verdict**: ⚠️ PROCEED WITH UPDATES + +--- + +## Executive Summary + +This is a small, well-scoped, low-risk feature backed by a detailed PRD (IHS-259) and a clear parent effort (INFP-636). The spec and plan are strong: the problem is real and clearly stated, backwards compatibility is treated as a first-class P1 requirement, async/sync parity is enforced, and the technical approach is grounded in existing prior art (`X-Infrahub-Tracker`) rather than inventing a new mechanism. There are **no Must-Address blockers**. Three low-risk clarifications are worth applying before task generation: (1) SC-002's "byte-for-byte identical" is not literally testable given transport-injected headers and should be reframed as "`X-Priority` absent, no other SDK-set header changed"; (2) batch-mode default inheritance is claimed in scope but has no verifying success criterion or test; (3) the `all` pagination loop must forward `priority` on every page request — worth stating explicitly so it isn't missed. All three are applied as edits below. + +--- + +## Product Lens Findings 🎯 + +### Problem Validation + +| ID | Severity | Finding | Suggestion | +|----|----------|---------|------------| +| P1 | ✅ (strength) | Problem statement is clear, evidence is architectural (shared worker pool / no interactive-vs-background signal), and it is correctly framed as the SDK's slice of INFP-636 with server work explicitly out of scope. | None — keep as is. | + +### User Value Assessment + +| ID | Severity | Finding | Suggestion | +|----|----------|---------|------------| +| P2 | 💡 | The PRD's 7 user stories were consolidated into 5 spec stories. This is reasonable grouping (enum-typing folded into US1/US2), not drift, but the "typed enum instead of raw strings" value (PRD story 2) is now implicit. | Confirmed acceptable; enum value is captured in FR-001 and the API contract. No change required. | + +### Alternative Approaches + +| ID | Severity | Finding | Suggestion | +|----|----------|---------|------------| +| P3 | 🤔 | A `Literal["high","normal","low"]` would be simpler than an enum. | Rejected by design — FR-001/PRD mandate a `Priority` enum for typo-safety and intent expression. No action. | + +### Edge Cases & UX + +| ID | Severity | Finding | Suggestion | +|----|----------|---------|------------| +| P4 | 💡 | The "caller manually pre-populates an `X-Priority` header" edge case is only reachable via the low-level `_get`/`_post` `headers=` kwarg — the covered public methods don't expose a raw headers argument. As written it reads as broader than it is. | Minor wording clarification (applied): scope this edge case to the low-level transport methods. | + +### Success Measurement + +| ID | Severity | Finding | Suggestion | +|----|----------|---------|------------| +| P5 | 💡 | Batch mode is listed under scope ("inherits the client default") but no Success Criterion asserts it, so "does the default actually ride batch requests?" is unverified. | Add a validation note/task asserting batch requests carry the configured default (applied to plan/spec + tasks). | + +--- + +## Engineering Lens Findings 🔬 + +### Architecture Soundness + +| ID | Severity | Finding | Suggestion | +|----|----------|---------|------------| +| E1 | ✅ (strength) | Injecting the default into base `self.headers` and applying the override only in the two `execute_graphql*` funnels is the right factoring — it guarantees transport coverage without touching ~10 call sites and mirrors `X-INFRAHUB-KEY`. | None. | + +### Failure Mode Analysis + +| ID | Severity | Finding | Suggestion | +|----|----------|---------|------------| +| E2 | 💡 | `all()` paginates by calling `execute_graphql` once per page (via `filters`). If `priority` is forwarded to only the first page, later pages would silently drop it. | State explicitly that the pagination loop forwards `priority` to every page call; cover with a multi-page test (applied). | +| E3 | 💡 | The multipart path (`_execute_graphql_with_file`) pops `content-type`; the override is applied to the copied dict, so `X-Priority` survives — but this ordering is load-bearing and easy to regress. | Add an explicit multipart override test and a note that `X-Priority` must be set after the content-type pop (applied to research/tasks). | + +### Security & Privacy + +| ID | Severity | Finding | Suggestion | +|----|----------|---------|------------| +| E4 | 🤔 | The header is client-asserted and trivially spoofable/escalatable by a caller. | Correctly out of scope — anti-escalation is a server/usage concern (INFP-636), already noted in spec Out of Scope. No action. | + +### Performance & Scalability + +| ID | Severity | Finding | Suggestion | +|----|----------|---------|------------| +| E5 | ✅ | One dict insertion per request; no new round-trips or allocation of concern. | None. | + +### Testing Strategy + +| ID | Severity | Finding | Suggestion | +|----|----------|---------|------------| +| E6 | 💡 | SC-002 says the unconfigured case is "asserted **byte-for-byte** against current behaviour." httpx injects its own headers (host, user-agent, content-length, etc.), so a literal byte-for-byte comparison of the full request is neither stable nor what the test should assert. | Reframe SC-002 as "no `X-Priority` header is emitted and no other SDK-set header changes," asserted via `"x-priority" not in request.headers` (applied to spec). | + +### Operational Readiness + +| ID | Severity | Finding | Suggestion | +|----|----------|---------|------------| +| E7 | 🤔 | No observability of which priority a request carried. | Out of scope for an SDK header emitter; the server observes the header. No action. | + +### Dependencies & Integration + +| ID | Severity | Finding | Suggestion | +|----|----------|---------|------------| +| E8 | ✅ | No new dependencies; reuses pydantic + httpx. `Config.clone()` carries the new field automatically. | None. | + +--- + +## Cross-Lens Insights + +| ID | Lens | Severity | Category | Finding | Suggestion | +|----|------|----------|----------|---------|------------| +| X1 | Both | 💡 | Scope × Verification | Batch mode + blob transports are claimed to inherit the client default (product scope) but the engineering verification for batch is absent (E2/P5 converge). | Add explicit tests that a configured default rides batch and blob requests; keeps the "every transport" promise honest. Applied to spec SC + tasks. | + +--- + +## Findings Summary Table + +| ID | Lens | Severity | Category | Finding | Suggestion | +|----|------|----------|----------|---------|------------| +| P1 | Product | ✅ | Problem Validation | Clear, evidence-backed problem | None | +| P2 | Product | 💡 | User Value | 7→5 story consolidation; enum value implicit | Confirmed OK; captured in FR-001 | +| P3 | Product | 🤔 | Alternatives | Literal vs enum | Enum mandated; no action | +| P4 | Product | 💡 | Edge/UX | Manual-header edge case broader than reachable | Scope wording to low-level methods (applied) | +| P5 | Product | 💡 | Success Measurement | Batch inheritance unverified | Add batch test/criterion (applied) | +| E1 | Engineering | ✅ | Architecture | Base-header + funnel factoring is sound | None | +| E2 | Engineering | 💡 | Failure Modes | Pagination must forward priority per page | State + test (applied) | +| E3 | Engineering | 💡 | Failure Modes | Multipart content-type pop ordering is load-bearing | Order note + test (applied) | +| E4 | Engineering | 🤔 | Security | Header spoofable | Out of scope; no action | +| E5 | Engineering | ✅ | Performance | Negligible overhead | None | +| E6 | Engineering | 💡 | Testing | "byte-for-byte" not literally testable | Reframe SC-002 (applied) | +| E7 | Engineering | 🤔 | Ops | No priority observability | Out of scope; no action | +| E8 | Engineering | ✅ | Dependencies | No new deps; clone carries field | None | +| X1 | Both | 💡 | Scope × Risk | Batch/blob inheritance unverified | Add tests (applied) | + +--- + +## Verdict & Remediation + +**⚠️ PROCEED WITH UPDATES** — no Must-Address blockers. Six 💡 recommendations, all clear and low-risk, are applied autonomously (per the prep workflow): + +1. **SC-002 reworded** (E6) — from "byte-for-byte" to "no `X-Priority` emitted; no other SDK-set header changes." +2. **New SC-006** (P5/X1) — a configured default rides batch and blob transfers. +3. **Edge case P4** — scoped the manual-header case to low-level `_get`/`_post`. +4. **Plan/research notes** (E2, E3) — pagination forwards `priority` per page; multipart sets `X-Priority` after the content-type pop. + +These are reflected in `spec.md` and `plan.md`/`research.md`. Tasks generation (Phase 4) will include the batch/blob/multipart/pagination tests. Ready for `/speckit-tasks`. diff --git a/dev/specs/ihs-259-sdk-x-priority-header/data-model.md b/dev/specs/ihs-259-sdk-x-priority-header/data-model.md new file mode 100644 index 000000000..47784bff7 --- /dev/null +++ b/dev/specs/ihs-259-sdk-x-priority-header/data-model.md @@ -0,0 +1,102 @@ +# Data Model: SDK `X-Priority` Request Header + +**Feature**: IHS-259 | **Date**: 2026-07-10 + +This feature introduces no persisted data. The "entities" are in-memory types and the wire header. Below: the new/changed types, their fields, validation rules, and the state/resolution logic. + +## Entity: `Priority` (new) + +A closed enumeration representing request priority. Owned by the SDK; no lifecycle beyond its value. + +| Member | Wire value | Meaning | +|----------|-----------|----------------------------------------------------------------| +| `HIGH` | `high` | Prefer this request; server should protect it under load. | +| `NORMAL` | `normal` | Default server treatment; equivalent to absent header. | +| `LOW` | `low` | Sheddable first; intended for background/bulk workloads. | + +- **Base type**: `str, enum.Enum` — each member *is* its lowercase wire token, so it drops directly into a header dict. +- **Location**: `infrahub_sdk/constants.py` (beside `InfrahubClientMode`). +- **Validation / coercion rules**: + - Exact string match: `Priority("low") == Priority.LOW`. + - Case-insensitive match via `_missing_`: `Priority("LOW")`, `Priority("Low")` → `Priority.LOW` (FR-002). + - Unknown value (`Priority("lowe")`) → `ValueError` (surfaced as `pydantic.ValidationError` at config load) (FR-007). +- **No ordering semantics**: the enum is a closed label set, not a comparable ranking. The SDK does not compare priorities; it only serialises the resolved value. + +## Entity: `Config.priority` (extended field) + +Extends `ConfigBase` (`infrahub_sdk/config.py`). + +| Attribute | Value | +|-----------|-------| +| Field name | `priority` | +| Type | `Priority \| None` | +| Default | `None` (no client-wide default → header omitted) | +| Env var | `INFRAHUB_PRIORITY` (via `env_prefix="INFRAHUB_"`) | +| Accepts | a `Priority` value, or a case-insensitive string (`"high"/"normal"/"low"`, any case) | +| Validation | pydantic + `Priority` enum; unknown value → `ValidationError` at load time | +| Carried by `clone()` | Yes (automatic — `clone()` iterates `Config.model_fields`) | + +- **Meaning of values**: + - `None` → no client-wide default; no header unless a per-request override is given. + - `Priority.X` → every request from this client carries `X-Priority: x` unless overridden per request. + +## Entity: Base request headers `self.headers` (extended) + +The client's base header dict, built once in `BaseClient.__init__` (`infrahub_sdk/client.py`). + +- **Before**: `{"content-type": "application/json"[, "X-INFRAHUB-KEY": ]}`. +- **After**: additionally `"X-Priority": ` **iff** `config.priority is not None`. +- **Invariant (FR-004/SC-002)**: when `config.priority is None`, the dict is byte-for-byte what it is today — no `X-Priority` key. +- Every transport copies this dict per request, so the default rides all of them. + +## Entity: `X-Priority` wire header (new, external contract) + +| Attribute | Value | +|-----------|-------| +| Header name | `X-Priority` (exact) | +| Value | one of `high` / `normal` / `low` (lowercase) | +| Presence | present only when the resolved priority is non-`None` | +| Server semantics (per INFP-636) | case-insensitive; absent or unknown treated as `normal` | + +See [contracts/x-priority-header.md](./contracts/x-priority-header.md). + +## Resolution logic (the core rule) + +Per request, the emitted header is determined by: + +```text +resolved = per_request if per_request is not None else client_default +if resolved is None: omit the X-Priority header +else: send X-Priority: resolved.value +``` + +Realised in code as: the client default is already in the copied `self.headers`; then `if per_request is not None: headers["X-Priority"] = per_request.value`. + +### Resolution truth table + +| Client default | Per-request arg | Emitted header | +|----------------|-----------------|-----------------------| +| `None` | `None` | *(none)* | +| `None` | `HIGH` | `X-Priority: high` | +| `None` | `NORMAL` | `X-Priority: normal` | +| `LOW` | `None` | `X-Priority: low` | +| `LOW` | `HIGH` | `X-Priority: high` | +| `LOW` | `NORMAL` | `X-Priority: normal` (explicit step-up wins) | +| `NORMAL` | `None` | `X-Priority: normal` | +| `HIGH` | `LOW` | `X-Priority: low` | + +- There is no per-request way to force "send no header" once a default is set; passing `NORMAL` explicitly is the accepted equivalent (spec Edge Cases). +- A per-request value never mutates client state — the next un-annotated call reverts to the client default (SC-003). + +## Coverage of the per-request override + +| Surface | Client default rides it? | Per-request `priority=` override? | +|---------|--------------------------|-----------------------------------| +| `execute_graphql` + file variant | Yes | **Yes** | +| `get`, `all`, `filters`, `count` | Yes | **Yes** | +| diff methods (`create_diff`, `get_diff_summary`, `get_diff_tree`) | Yes | **Yes** | +| node `save` / `create` / `update` / `delete` | Yes | **Yes** (forwarded) | +| resource-pool peer fetch within a node create/update | Yes | **Yes** (inherits the operation's `priority`) | +| `client.create` (builds an unsaved node, issues no request) | n/a | No (nothing to send) | +| raw blob `_get` / `_post` / `_get_streaming` | Yes | No (v1) | +| batch mode | Yes | No (v1) | diff --git a/dev/specs/ihs-259-sdk-x-priority-header/opsmill-implement-report.md b/dev/specs/ihs-259-sdk-x-priority-header/opsmill-implement-report.md new file mode 100644 index 000000000..6e82882d1 --- /dev/null +++ b/dev/specs/ihs-259-sdk-x-priority-header/opsmill-implement-report.md @@ -0,0 +1,106 @@ +# Implementation Report: SDK `X-Priority` Request Header (IHS-259) + +**Status**: ✅ DONE + +## 1. Header + +- **Feature**: SDK `X-Priority` request header (client-wide default + per-request override). +- **Spec dir**: `specs/ihs-259-sdk-x-priority-header/` +- **Base commit**: `6b82d0f` (prep artifacts; tail started here) +- **Head commit**: `2cfe193` +- **Branch**: `dga/feat-x-priority-aa2nd` +- **Tasks**: 35/35 complete (all `[X]`). +- **Wall-clock**: ~1h (implement loop + review + fixes). + +## 2. Chunk-by-chunk ledger + +| # | Chunk (phase) | Tasks | ✅ / ⚠️ / ❌ | Commit(s) | Notes | +|---|---------------|-------|--------------|-----------|-------| +| 0 | Phase 1 Setup (T001) | 1 | 1 / 0 / 0 | `f4422e5` | Run by orchestrator as preflight; baseline 107 passed. | +| 1 | Phase 2 Foundational (T002–T004) | 3 | 3 / 0 / 0 | `c95dc7a` | `Priority` enum + `_missing_`, export, `Config.priority`. Import smoke-check passed. | +| 2 | Phase 3 US1 (T005–T010) | 6 | 6 / 0 / 0 | `feef536` | Base-header injection; default rides GraphQL/multipart/blob/batch. 14 tests. | +| 3 | Phase 4 US3 (T011–T012) | 2 | 2 / 0 / 0 | `39b566e` | No-header-when-unconfigured; no production change needed. | +| 4 | Phase 5 US2 (T013–T024) | 12 | 10 / 2 / 0 | `af1e6c4` | Per-request override on funnels + high-level + node. **T016/T018 ⚠️ partial**: `client.create` intentionally excluded (issues no request; covered at `node.save`). **Load-bearing merge flip** applied (see §6). 40 tests. | +| 5 | Phase 6 US4 (T025–T027) | 3 | 3 / 0 / 0 | `3feedaa` | Config validation (case-insensitive accept, reject unknown, default None). Justified `# ty: ignore` on deliberate string-coercion tests. | +| 6 | Phase 7 US5 (T028–T029) | 2 | 2 / 0 / 0 | `afbb902` | Parity audit (all wire tests already dual); resolution truth-table parity test (16 cases). | +| 7 | Phase 8 Polish (T030–T035) | 6 | 6 / 0 / 0 | `76b8834` | Docstrings, `docs-generate`+`docs-validate` (green), changelog `1151.added.md`, full-suite run. | + +**Review-driven commits** (Phase 6): `8ab2964` (HIGH fix), `2cfe193` (Medium test-gap closure). + +## 3. Tasks not completed + +None. All 35 tasks are `[X]`. + +- **Nuance (not incomplete)**: T016/T018 deliberately excluded `client.create` from the `priority=` kwarg. Reason (from the subagent, confirmed by the code review): `client.create` only constructs an unsaved `InfrahubNode` and issues no HTTP request — the create request is made by `node.save()`/`node.create()`, which DO carry `priority` (T019/T020, tested). Adding an unused kwarg to `client.create` would be a misleading no-op and a lint error. FR-005's "create" surface is therefore satisfied at the request-issuing layer. + +## 4. Local-pass evidence + +All tests added/modified by this run, observed passing locally (unit; project has no locally-runnable E2E — the "E2E scenario" in the PRD is realized as unit wire-assertions). Aggregated from chunk subagents; final consolidated run: `tests/unit/sdk/ → 1145 passed`. + +| Test id | Type | Run command | Passed at (ISO 8601) | Env | Verbatim pass line | +|---------|------|-------------|----------------------|-----|--------------------| +| `test_config.py::test_invalid_priority_rejected` | unit | `uv run pytest tests/unit/sdk/test_config.py -q` | 2026-07-11T14:47:13Z | n/a | `Pytest: 25 passed` | +| `test_config.py::test_priority_case_insensitive_acceptance` (12 params) | unit | `uv run pytest tests/unit/sdk/test_config.py -q` | 2026-07-11T14:47:13Z | n/a | `Pytest: 25 passed` | +| `test_config.py::test_priority_from_env_var` (3 params) | unit | `uv run pytest tests/unit/sdk/test_config.py -q` | 2026-07-11T14:47:13Z | n/a | `Pytest: 25 passed` | +| `test_config.py::test_priority_default_is_none` | unit | `uv run pytest tests/unit/sdk/test_config.py -q` | 2026-07-11T14:47:13Z | n/a | `Pytest: 25 passed` | +| `test_priority.py::test_priority_header_on_graphql_query` (×2 clients) | unit | `uv run pytest tests/unit/sdk/test_priority.py -q` | 2026-07-11T15:12:59Z | n/a | `Pytest: 64 passed` | +| `test_priority.py::test_priority_header_on_graphql_mutation` (×2) | unit | `uv run pytest tests/unit/sdk/test_priority.py -q` | 2026-07-11T15:12:59Z | n/a | `Pytest: 64 passed` | +| `test_priority.py::test_priority_header_on_blob_download` (×2) | unit | `uv run pytest tests/unit/sdk/test_priority.py -q` | 2026-07-11T15:12:59Z | n/a | `Pytest: 64 passed` | +| `test_priority.py::test_priority_header_on_blob_upload` (×2) | unit | `uv run pytest tests/unit/sdk/test_priority.py -q` | 2026-07-11T15:12:59Z | n/a | `Pytest: 64 passed` | +| `test_priority.py::test_priority_header_on_multipart_upload` (×2) | unit | `uv run pytest tests/unit/sdk/test_priority.py -q` | 2026-07-11T15:12:59Z | n/a | `Pytest: 64 passed` | +| `test_priority.py::test_priority_header_on_batched_requests` (×2) | unit | `uv run pytest tests/unit/sdk/test_priority.py -q` | 2026-07-11T15:12:59Z | n/a | `Pytest: 64 passed` | +| `test_priority.py::test_priority_normal_is_always_emitted` (×2) | unit | `uv run pytest tests/unit/sdk/test_priority.py -q` | 2026-07-11T15:12:59Z | n/a | `Pytest: 64 passed` | +| `test_priority.py::test_no_priority_header_on_graphql_when_unconfigured` (×2) | unit | `uv run pytest tests/unit/sdk/test_priority.py -q` | 2026-07-11T15:12:59Z | n/a | `Pytest: 64 passed` | +| `test_priority.py::test_no_priority_header_on_blob_download_when_unconfigured` (×2) | unit | `uv run pytest tests/unit/sdk/test_priority.py -q` | 2026-07-11T15:12:59Z | n/a | `Pytest: 64 passed` | +| `test_priority.py::test_no_priority_header_on_blob_upload_when_unconfigured` (×2) | unit | `uv run pytest tests/unit/sdk/test_priority.py -q` | 2026-07-11T15:12:59Z | n/a | `Pytest: 64 passed` | +| `test_priority.py::test_no_priority_header_on_multipart_upload_when_unconfigured` (×2) | unit | `uv run pytest tests/unit/sdk/test_priority.py -q` | 2026-07-11T15:12:59Z | n/a | `Pytest: 64 passed` | +| `test_priority.py::test_unconfigured_headers_unchanged_versus_baseline` (×2) | unit | `uv run pytest tests/unit/sdk/test_priority.py -q` | 2026-07-11T15:12:59Z | n/a | `Pytest: 64 passed` | +| `test_priority.py::test_override_on_no_default_client_then_no_leak` (×2) | unit | `uv run pytest tests/unit/sdk/test_priority.py -q` | 2026-07-11T15:12:59Z | n/a | `Pytest: 64 passed` | +| `test_priority.py::test_override_beats_default_then_reverts` (×2) | unit | `uv run pytest tests/unit/sdk/test_priority.py -q` | 2026-07-11T15:12:59Z | n/a | `Pytest: 64 passed` | +| `test_priority.py::test_override_normal_beats_low_default` (×2) | unit | `uv run pytest tests/unit/sdk/test_priority.py -q` | 2026-07-11T15:12:59Z | n/a | `Pytest: 64 passed` | +| `test_priority.py::test_override_on_get` (×2) | unit | `uv run pytest tests/unit/sdk/test_priority.py -q` | 2026-07-11T15:12:59Z | n/a | `Pytest: 64 passed` | +| `test_priority.py::test_override_on_all_carries_on_every_page` (×2) | unit | `uv run pytest tests/unit/sdk/test_priority.py -q` | 2026-07-11T15:12:59Z | n/a | `Pytest: 64 passed` | +| `test_priority.py::test_override_on_save_create_path` (×2) | unit | `uv run pytest tests/unit/sdk/test_priority.py -q` | 2026-07-11T15:12:59Z | n/a | `Pytest: 64 passed` | +| `test_priority.py::test_override_on_save_update_path` (×2) | unit | `uv run pytest tests/unit/sdk/test_priority.py -q` | 2026-07-11T15:12:59Z | n/a | `Pytest: 64 passed` | +| `test_priority.py::test_override_on_node_delete` (×2) | unit | `uv run pytest tests/unit/sdk/test_priority.py -q` | 2026-07-11T15:12:59Z | n/a | `Pytest: 64 passed` | +| `test_priority.py::test_override_on_diff_method` (create_diff, ×2) | unit | `uv run pytest tests/unit/sdk/test_priority.py -q` | 2026-07-11T15:12:59Z | n/a | `Pytest: 64 passed` | +| `test_priority.py::test_override_on_get_diff_summary` (×2) | unit | `uv run pytest tests/unit/sdk/test_priority.py -q` | 2026-07-11T15:12:59Z | n/a | `Pytest: 64 passed` | +| `test_priority.py::test_override_on_get_diff_tree` (×2) | unit | `uv run pytest tests/unit/sdk/test_priority.py -q` | 2026-07-11T15:12:59Z | n/a | `Pytest: 64 passed` | +| `test_priority.py::test_override_on_multipart_upload` (×2) | unit | `uv run pytest tests/unit/sdk/test_priority.py -q` | 2026-07-11T15:12:59Z | n/a | `Pytest: 64 passed` | +| `test_priority.py::test_resolution_truth_table_parity` (16 params) | unit | `uv run pytest tests/unit/sdk/test_priority.py -q` | 2026-07-11T15:12:59Z | n/a | `Pytest: 64 passed` | +| `test_relogin_headers.py::test_relogin_retry_uses_refreshed_auth_header` (×2) | unit | `uv run pytest tests/unit/sdk/test_relogin_headers.py -p no:cacheprovider` | 2026-07-11T15:08:18Z | n/a | `4 passed in 0.02s` | +| `test_relogin_headers.py::test_merge_request_headers_reasserts_live_auth` (×2) | unit | `uv run pytest tests/unit/sdk/test_relogin_headers.py -p no:cacheprovider` | 2026-07-11T15:08:18Z | n/a | `4 passed in 0.02s` | + +**Consolidated final runs**: `tests/unit/sdk/ → 1145 passed` (2026-07-11); full `tests/unit/ → 1588 passed, 8 failed, 1 xfailed` — the 8 failures are **pre-existing** (`ctl/` menu/repo/schema/task app CLI-rendering + `pytest_plugin` fixture), fail identically on base `6b82d0f`, and none reference priority. + +## 5. Review findings + +Dual-lens review (correctness/types/errors, tests, comments/simplify) across `6b82d0f..HEAD`. + +| Severity | File | Summary | Disposition | +|----------|------|---------|-------------| +| 🔴 High | `client.py` (8 transport helpers) | Merge-order flip let a stale per-request header snapshot overwrite the freshly-refreshed `Authorization` on the relogin retry → password-auth token-refresh broken. | **Fixed inline** (`8ab2964`) — added `BaseClient._merge_request_headers` re-asserting live auth after the per-request merge; regression test proves it (removing fix → 4 failures). | +| 🟡 Medium | `test_priority.py` | Node `delete()` per-request override untested on the wire. | **Fixed inline** (`2cfe193`). | +| 🟡 Medium | `test_priority.py` | `save()` update-path override untested (only create branch). | **Fixed inline** (`2cfe193`). | +| 🟢 Low | `test_priority.py` | Only `create_diff` of the 3 diff methods tested. | **Fixed inline** (`2cfe193`) — added summary + tree. | +| 🟢 Low | `client.py` | `get`/`create_diff`/`get_diff_summary` have no docstring, so `priority` is undocumented there (pre-existing lack of docstrings; feature widens the gap). | **Deferred** — cosmetic; matches existing docstring density. | +| 🟢 Low | `node.py` | Node docstrings omit the "when None → client default" sentence the client docstrings include. | **Deferred** — accurate, just terser. | +| 🟢 Low (advisory) | `client.py` | `if priority is not None: headers[...] = ...` duplicated across 4 funnels; a helper could remove it + the `# noqa: PLR0912`. | **Deferred** — advisory; async/sync split limits gains. | + +Positive observations from review: `Priority._missing_` correct (case-insensitive, no recursion, non-str safe); resolution applied exactly once per funnel; async/sync parity complete; type hints clean; no silent failures; test assertions are genuinely on-the-wire (a reverted feature would fail the mocks), parity is real (sync path exercised), multi-page `all` truly asserts every page. + +## 6. Autonomous decisions + +1. **Ran T001 in the orchestrator** (env sync + baseline) rather than dispatching a subagent for a one-line check; ticked it with a fixup commit. +2. **`client.create` excluded from `priority=`** (T016/T018 ⚠️): it issues no request; the create request path (`node.save`/`create`) carries priority and is tested. Review confirmed this is correct and no request-issuing FR-005 surface was missed. +3. **Merge-order flip** (chunk 4): the subagent discovered the transport helpers let base `self.headers` overwrite per-request headers (defeating the override) and flipped precedence. This was load-bearing and correct for the feature — but the review then caught that it broke the relogin auth-refresh path; the final fix (`_merge_request_headers`) preserves BOTH invariants (per-request wins for non-auth keys; live auth always wins). Full suite confirms no regression. +4. **Closed two Medium + one Low test gap inline** (`2cfe193`) although the workflow only mandates inline fixes for High+. Rationale: these are the core override surfaces; an untested forwarding call site could silently regress. Cheap, low-risk, high-value. No production bug was found while doing so. +5. **Deferred the Low docstring/simplify findings** — cosmetic, no correctness impact; recorded above for a follow-up. +6. **`docs-generate` drift**: regeneration also touched 8 unrelated `.mdx` files already stale vs. the generator; all regenerated output was committed so `docs-validate` stays green (the only way to keep it passing). + +## 7. Suggested next steps + +1. **Open a PR** for `dga/feat-x-priority-aa2nd` → `stable` (this is a public-API change per IHS-259 governance; the PR description should call that out). +2. (Optional) Address the deferred Low findings: add one-line docstrings with the `priority` Args entry to `get`/`create_diff`/`get_diff_summary`, and consider the `_apply_priority` helper to drop the two `# noqa: PLR0912`. +3. The 8 pre-existing `ctl`/`pytest_plugin` unit failures are unrelated to this feature but exist on `stable`'s merge-base — worth a separate ticket if not already tracked. +4. Run `speckit-opsmill-extract` if you want ADRs/guidelines mined from this spec dir. diff --git a/dev/specs/ihs-259-sdk-x-priority-header/plan.md b/dev/specs/ihs-259-sdk-x-priority-header/plan.md new file mode 100644 index 000000000..23fae45f2 --- /dev/null +++ b/dev/specs/ihs-259-sdk-x-priority-header/plan.md @@ -0,0 +1,115 @@ +# Implementation Plan: SDK `X-Priority` Request Header + +**Branch**: `dga/feat-x-priority-aa2nd` | **Date**: 2026-07-10 | **Spec**: [spec.md](./spec.md) + +**Input**: Feature specification from `specs/ihs-259-sdk-x-priority-header/spec.md` + +## Summary + +Add a first-class request-priority concept to the SDK, emitted as an `X-Priority: high|normal|low` HTTP header. Two configuration surfaces: a client-wide default via `Config.priority` (rides every transport by being injected into the client's base `self.headers`) and a per-request `priority=` keyword argument on the covered public methods (resolved as `per_request if per_request is not None else client_default`). When nothing is configured, no header is emitted — byte-for-byte identical to today. Both `InfrahubClient` (async) and `InfrahubClientSync` (sync) behave identically. No server-side logic, no 429 handling. + +**Technical approach** (grounded in the existing `X-Infrahub-Tracker` prior art): + +1. New `Priority(str, enum.Enum)` in `infrahub_sdk/constants.py` with members `HIGH="high"`, `NORMAL="normal"`, `LOW="low"` and a case-insensitive `_missing_` classmethod. +2. New `Config.priority: Priority | None = None` field (auto-binds to `INFRAHUB_PRIORITY` via the existing `env_prefix`). Pydantic + the enum give validation and case-insensitive string coercion for free; unknown values raise at config load. +3. Inject the configured default once into `BaseClient.__init__`'s base `self.headers` (right next to the `X-INFRAHUB-KEY` line). Because every transport already merges `self.headers`, the default automatically rides GraphQL, multipart upload, and raw blob `_get`/`_post`. +4. Add `priority: Priority | None = None` to `execute_graphql` and `_execute_graphql_with_file` (async + sync). These are the single points where the per-request header is applied: `if priority is not None: headers["X-Priority"] = priority.value` on the already-copied header dict — which realises the resolution rule exactly (a `None` per-request keeps whatever the base default was; an explicit value, including `NORMAL`, overrides it). +5. Thread the `priority` kwarg through the higher-level callers so they forward it to the two execute methods: client `get`, `all` (via `filters`), `create`, `create_diff`/`get_diff_summary`/`get_diff_tree`; node `save`/`create`/`update`/`delete`. Raw blob `_get`/`_post` and batch mode inherit the client default only (no per-call override in v1). + +**Two load-bearing details surfaced by the critique** (see [critiques/](./critiques/)): + +- **Pagination**: `all()` calls `execute_graphql` once per page via `filters`. The `priority` kwarg must be forwarded on **every** page request, not just the first — covered by a multi-page test. +- **Multipart ordering**: `_execute_graphql_with_file` pops `content-type` from the copied header dict for multipart. `X-Priority` must be applied **after** that pop (and the default already in `self.headers` survives it, since only `content-type` is removed). Covered by an explicit multipart override test. +- **Batch/blob inheritance**: verified by test (SC-006) — a configured default must ride batch-mode and raw blob requests even though neither exposes a per-request override. + +## Technical Context + +**Language/Version**: Python 3.10–3.13 + +**Primary Dependencies**: pydantic >=2.0, pydantic-settings, httpx, graphql-core + +**Storage**: N/A (stateless HTTP header) + +**Testing**: pytest, pytest-httpx (`HTTPXMock` with `match_headers=`), pytest async auto-mode + +**Target Platform**: Library consumed by network-automation code, `infrahubctl`, and the Infrahub Ansible collection + +**Project Type**: Single-project Python library (async + sync dual client) + +**Performance Goals**: No measurable overhead — a single dict insertion per request; no new network round-trips + +**Constraints**: Zero behaviour change when unconfigured (byte-for-byte identical headers); async/sync parity; no new dependencies; no changes to generated `protocols.py` + +**Scale/Scope**: ~1 new enum, 1 new config field, header injection in `BaseClient.__init__`, and a `priority` kwarg on ~10 public methods across two clients plus the node module + +## Constitution Check + +*GATE: Must pass before Phase 0 research. Re-check after Phase 1 design.* + +The project constitution (`.specify/memory/constitution.md`) is an unpopulated template with no ratified principles, so there are no formal constitutional gates to evaluate. In its place, the binding constraints are the repository's `AGENTS.md` boundaries: + +- **Async/sync dual pattern (Always)** — satisfied: every change is applied to both clients; FR-008 / User Story 5 make parity a first-class, tested requirement. +- **Type hints on all signatures (Always)** — satisfied: the new enum and every touched signature are fully typed (`Priority | None`). +- **Do not modify generated `protocols.py` (Never)** — satisfied: no generated code is touched. +- **No new dependencies (Ask first)** — satisfied: none added. +- **Changing public API signatures (Ask first)** — this feature *intentionally* changes public signatures (new enum, new config field, new `priority` kwarg). This is the explicit, PRD-approved purpose of the ticket (governance gate checked in IHS-259); documented as an accepted assumption in the spec. +- **Docs regeneration (Always, after config/docstring changes)** — handled: `uv run invoke docs-generate` is a task in Phase 4. + +**Result**: PASS (no unjustified violations). Re-checked post-design: unchanged. + +## Project Structure + +### Documentation (this feature) + +```text +specs/ihs-259-sdk-x-priority-header/ +├── plan.md # This file +├── research.md # Phase 0 output +├── data-model.md # Phase 1 output +├── quickstart.md # Phase 1 output +├── contracts/ # Phase 1 output (public API + wire contract) +│ ├── priority-api.md +│ └── x-priority-header.md +├── checklists/ +│ └── requirements.md # From /speckit-specify +└── tasks.md # Phase 2 output (/speckit-tasks) +``` + +### Source Code (repository root) + +```text +infrahub_sdk/ +├── constants.py # ADD: Priority(str, enum.Enum) + case-insensitive _missing_ +├── config.py # ADD: Config.priority field (ConfigBase), imports Priority +├── client.py # EDIT: BaseClient.__init__ base-header injection; +│ # priority kwarg on execute_graphql + _execute_graphql_with_file +│ # (async + sync); thread through get/all/filters/create/diff methods +└── node/ + └── node.py # EDIT: priority kwarg on save/create/update/delete (async + sync), + # forwarded to client execute methods + +tests/unit/sdk/ +├── test_config.py # ADD: Config.priority validation (enum/string/case/reject-unknown) +├── test_priority.py # ADD (new): Priority enum + resolution behaviour +├── test_client.py # ADD: header-on-the-wire assertions (default, override, omit) both clients +├── test_object_store.py # ADD: blob transports carry the default +└── conftest.py # reuse BothClients fixture pattern + +docs/ # regenerated via `uv run invoke docs-generate` +``` + +**Structure Decision**: Single-project library layout (the existing SDK structure). All production changes are confined to `constants.py`, `config.py`, `client.py`, and `node/node.py`; no new modules are required beyond the enum, which lives with the other client enums in `constants.py`. + +## Design Decisions (detail) + +See [research.md](./research.md) for the full decision log. Highlights: + +- **Config field is named `priority`, not `x_priority`** — the PRD specifies `Config.priority`; the `X-` prefix belongs to the wire header, not the config surface. Auto-binds to `INFRAHUB_PRIORITY`. +- **Default injected into base `self.headers`, not at each call site** — this is the mechanism that guarantees FR-003 (every transport) without touching blob/batch code paths, and mirrors how `X-INFRAHUB-KEY` already rides every request. +- **Per-request application lives in `execute_graphql` / `_execute_graphql_with_file` only** — every high-level method funnels through these two, so the resolution rule is implemented once per client (twice total) instead of at ~10 call sites. Higher-level methods only *forward* the kwarg. +- **Resolution is realised by override-if-present on the copied header dict** — `copy.copy(self.headers)` already carries the default; `if priority is not None: headers["X-Priority"] = priority.value` yields exactly `per_request if per_request is not None else client_default`, including the explicit-`NORMAL`-beats-`low`-default edge case. +- **Case-insensitivity via `Priority._missing_`** — handles `LOW`/`Low`/`low` from env/file config and raises for unknown values, satisfying FR-002 and FR-007 with no bespoke validator. + +## Complexity Tracking + +No constitutional violations to justify — this section is intentionally empty. diff --git a/dev/specs/ihs-259-sdk-x-priority-header/quickstart.md b/dev/specs/ihs-259-sdk-x-priority-header/quickstart.md new file mode 100644 index 000000000..241b370dc --- /dev/null +++ b/dev/specs/ihs-259-sdk-x-priority-header/quickstart.md @@ -0,0 +1,93 @@ +# Quickstart / Validation Guide: SDK `X-Priority` Request Header + +**Feature**: IHS-259 | **Date**: 2026-07-10 + +How to exercise and validate the feature end-to-end. See [contracts/priority-api.md](./contracts/priority-api.md) for the API surface and [data-model.md](./data-model.md) for the resolution truth table. + +## Prerequisites + +```bash +uv sync --all-groups --all-extras +``` + +## Usage examples (what the feature enables) + +### Client-wide default (P1) + +```python +from infrahub_sdk import InfrahubClient, Config +from infrahub_sdk.constants import Priority + +# A client dedicated to background work tags every request low. +client = InfrahubClient(config=Config(address="http://localhost:8000", priority=Priority.LOW)) + +# Every request below carries `X-Priority: low` with no call-site changes: +await client.all(kind="BuiltinTag") # GraphQL +await client.execute_graphql(query=MY_QUERY) # GraphQL +# ...multipart uploads and blob get/post from this client also carry it. +``` + +Via environment / file config (case-insensitive): + +```bash +export INFRAHUB_PRIORITY=LOW # accepted; normalised to Priority.LOW +``` + +### Per-request override (P2) + +```python +client = InfrahubClient(config=Config(address="http://localhost:8000", priority=Priority.LOW)) + +# This one user-triggered call steps up to high; the rest stay low. +node = await client.get(kind="BuiltinTag", name__value="blue", priority=Priority.HIGH) + +# Explicit NORMAL beats a LOW default for this call only: +await client.execute_graphql(query=MY_QUERY, priority=Priority.NORMAL) # -> X-Priority: normal +``` + +### Zero behaviour change when unconfigured (P1) + +```python +client = InfrahubClient(config=Config(address="http://localhost:8000")) # no priority +await client.all(kind="BuiltinTag") # NO X-Priority header — identical to pre-feature SDK +``` + +### Invalid value rejected at config load (P2) + +```python +Config(address="http://localhost:8000", priority="lowe") # raises pydantic.ValidationError +``` + +## Validation scenarios (map to Success Criteria) + +Run the unit suite: + +```bash +uv run pytest tests/unit/sdk/test_priority.py tests/unit/sdk/test_config.py \ + tests/unit/sdk/test_client.py tests/unit/sdk/test_object_store.py -q +``` + +| Scenario | How it is asserted | Criterion | +|----------|--------------------|-----------| +| Default rides GraphQL, multipart, blob | `httpx_mock.add_response(match_headers={"X-Priority": "low"})` for each transport; request only matches if header present | SC-001 | +| Unconfigured client emits no header | capture request via `httpx_mock.get_requests()`, assert `"x-priority" not in request.headers` | SC-002 | +| Per-request override, then revert | override call matches `{"X-Priority": "high"}`; next un-annotated call matches the default (or no header) | SC-003 | +| Explicit `NORMAL` beats `LOW` default | override call matches `{"X-Priority": "normal"}` | SC-003 (edge) | +| Invalid value rejected | `pytest.raises(pydantic.ValidationError, match=...)` on `Config(priority="lowe")` | SC-004 | +| Case-insensitive config accepted | `Config(priority="LOW").priority is Priority.LOW` | FR-002 | +| Async/sync parity | parametrize every wire test over `["standard", "sync"]` via the `BothClients` fixture | SC-005 | + +## Full quality gate (run before commit) + +```bash +uv run invoke format lint-code +uv run invoke docs-generate # required: new Config field + docstrings +uv run invoke docs-validate +uv run pytest tests/unit/ +``` + +## Expected outcomes + +- All new unit tests pass for both `standard` and `sync` clients. +- `docs-validate` passes (generated docs include the new `Config.priority` field). +- No change to any test that exercises an unconfigured client (backwards compatibility intact). diff --git a/dev/specs/ihs-259-sdk-x-priority-header/research.md b/dev/specs/ihs-259-sdk-x-priority-header/research.md new file mode 100644 index 000000000..e4b1ea6ee --- /dev/null +++ b/dev/specs/ihs-259-sdk-x-priority-header/research.md @@ -0,0 +1,83 @@ +# Research: SDK `X-Priority` Request Header + +**Feature**: IHS-259 | **Date**: 2026-07-10 + +This feature has no external unknowns — the wire contract is fixed by INFP-636 and the PRD. "Research" here is the codebase investigation that fixes the *how*. Each decision below is grounded in existing code (`file:line` references from the current tree). + +## Decision 1 — Where the `Priority` enum lives and its shape + +- **Decision**: `class Priority(str, enum.Enum)` in `infrahub_sdk/constants.py`, members `HIGH = "high"`, `NORMAL = "normal"`, `LOW = "low"`, plus a case-insensitive `_missing_` classmethod. +- **Rationale**: `constants.py` already hosts `InfrahubClientMode(str, enum.Enum)` (`constants.py:4`) and is already imported by both `config.py` (`config.py:11`) and `client.py`. A `str`-valued enum means `Priority.LOW.value == "high"`-style access gives the exact wire token, and the member *is* a `str` so it slots straight into a headers dict. `_missing_` lets `Priority("LOW")` resolve case-insensitively, which pydantic uses when coercing env/file strings. +- **Alternatives considered**: + - `infrahub_sdk/enums.py` (`OrderDirection`, `enums.py:4`) — viable, but `constants.py` is the home for *client/config-consumed* enums (`InfrahubClientMode`), which is exactly this case. + - A `Literal["high","normal","low"]` instead of an enum — rejected: FR-001 explicitly requires a `Priority` enum so callers write `Priority.LOW` and cannot typo the contract. + +```python +# infrahub_sdk/constants.py +class Priority(str, enum.Enum): + HIGH = "high" + NORMAL = "normal" + LOW = "low" + + @classmethod + def _missing_(cls, value: object) -> "Priority | None": + if isinstance(value, str): + for member in cls: + if member.value == value.lower(): + return member + return None +``` + +## Decision 2 — Config field name, type, and validation + +- **Decision**: add `priority: Priority | None = Field(default=None, description=...)` to `ConfigBase` in `config.py`. No custom validator needed. +- **Rationale**: `ConfigBase(BaseSettings)` uses `model_config = SettingsConfigDict(env_prefix="INFRAHUB_", ...)` (`config.py:39`), so the field auto-binds to `INFRAHUB_PRIORITY`. Pydantic v2 coerces an incoming string to the enum, invoking `Priority._missing_` for case-insensitive matches (FR-002) and raising `ValidationError` for unknown values (FR-007). Default `None` means "no header" (FR-004). Existing enum fields (`mode`, `config.py:57`; `transport`, `config.py:87`) confirm the pattern; the closest optional-typed field is `api_token: str | None` (`config.py:41`). +- **Config field is `priority`, not `x_priority`**: the PRD names `Config.priority`; the `X-` prefix is a wire concern only. +- **`clone()` (`config.py:278`)** iterates `Config.model_fields` for any field not explicitly listed, so `priority` is carried into clones automatically — no change required there. +- **Alternatives considered**: a `@field_validator("priority", mode="before")` to lowercase strings — redundant once `_missing_` handles case; rejected to keep validation in one place (the enum). + +## Decision 3 — How the client-wide default rides every transport + +- **Decision**: inject the default into the base header dict in `BaseClient.__init__`, adjacent to the existing auth header: + +```python +# infrahub_sdk/client.py (BaseClient.__init__, ~line 218, after X-INFRAHUB-KEY) +if self.config.priority is not None: + self.headers["X-Priority"] = self.config.priority.value +``` + +- **Rationale**: every transport path copies and re-merges `self.headers` before sending — `execute_graphql` (`client.py:1244`), `_execute_graphql_with_file` (`client.py:1329`), `_post` (`client.py:1447`), `_get` (`client.py:1470`), `_get_streaming` (`client.py:1497`), `_post_multipart` (`client.py:1374`), and the object-store paths (`object_store.py:46,69,98,141,164,193`). Putting the default in `self.headers` once means it automatically covers GraphQL, multipart upload, and raw blob `_get`/`_post` (FR-003, SC-001) — including batch and blob paths that get no per-call override — with no per-call-site edits. This is exactly how `X-INFRAHUB-KEY` (`client.py:217-218`) already behaves. +- **Alternatives considered**: adding the header at each of the ~10 call sites — rejected as error-prone and easy to miss a transport (the very failure FR-003 guards against). + +## Decision 4 — Where and how the per-request override is applied + +- **Decision**: add `priority: Priority | None = None` to `execute_graphql` and `_execute_graphql_with_file` (async at `client.py:1201`/`1290`, sync at `client.py:2181`/`2270`). Immediately after the existing `headers = copy.copy(self.headers or {})` + tracker block, add: + +```python +if priority is not None: + headers["X-Priority"] = priority.value +``` + +- **Rationale**: these two methods are the single funnel for all GraphQL traffic. `copy.copy(self.headers)` already carries the client default, so `if priority is not None: headers["X-Priority"] = priority.value` computes exactly `resolved = per_request if per_request is not None else client_default` (FR-006): `None` keeps the default (or absence); an explicit value overrides it, including an explicit `NORMAL` stepping *up* from a `low` default (spec edge case, SC-003). Implementing it here means the rule exists twice (async + sync), not at every public method. +- **Alternatives considered**: a shared `_apply_priority(headers, priority)` helper — optional nicety; the two-line inline form mirrors the surrounding tracker code and is clearer in context. Left to implementer discretion; both satisfy the contract. + +## Decision 5 — Threading the kwarg through higher-level methods + +- **Decision**: higher-level methods gain `priority: Priority | None = None` and forward it to the execute methods; they do **not** re-implement resolution. + - Client: `get` (`client.py:442`), `all`→`filters` (`client.py:905`/`1131`), `create` (`client.py:400`), `create_diff` (`client.py:1695`), `get_diff_summary` (`client.py:1724`), `get_diff_tree` (`client.py:1763`) — and every sync twin. + - Node: `save` (`node.py:1241`), `create` (`node.py:1602`), `update` (`node.py:1681`), `delete` (`node.py:1214`) — plus sync twins — forward `priority` into `execute_graphql` / `_execute_graphql_with_file`. +- **Rationale**: `get`/`all`/`create` already funnel through `execute_graphql` carrying a `tracker`; adding a parallel `priority` passthrough matches the established shape. Node mutation methods already call the two execute methods (`node.py:1657-1678`), so they only need to forward the new kwarg. +- **Out of scope (v1)**: raw `_get`/`_post`/`_get_streaming` and batch mode expose no per-call override — they inherit the default from `self.headers` (spec Out of Scope + Edge Cases). No `priority` kwarg is added to those. +- **Pagination caveat (from critique E2)**: `all()` renders and calls `execute_graphql` per page (`client.py:1131` / sync `client.py:2907`). Forward `priority` inside the pagination loop so every page request carries it, not only page 1. Cover with a multi-page test. +- **Multipart ordering (from critique E3)**: in `_execute_graphql_with_file`, the existing code pops `content-type` from the copied headers for multipart. Apply the `X-Priority` override **after** the copy/pop so it is not lost; the base default in `self.headers` is unaffected because only `content-type` is removed. + +## Decision 6 — Testing strategy + +- **Decision**: mirror the `X-Infrahub-Tracker` test style — `pytest-httpx`'s `HTTPXMock.add_response(match_headers={"X-Priority": "low"})`, which only matches when the outgoing request carries that header; and negative assertions that no `X-Priority` is present when unconfigured. Parametrize over `["standard", "sync"]` via the existing `BothClients` fixture (`tests/unit/sdk/conftest.py:33-45`) for parity (FR-008/SC-005). +- **Rationale**: `match_headers` is the repo-standard way to assert on outgoing headers (`test_object_store.py:22-29`, `test_client.py:366+`, `test_diff_summary.py:92+`). The `BothClients` fixture is the repo-standard parity harness. +- **Byte-for-byte no-header check (SC-002)**: assert absence via a request captured by the mock (`httpx_mock.get_requests()` → `"x-priority" not in request.headers`), following the direct-header-read style at `test_rate_limit_retry.py:677`. +- **Config validation (SC-004)**: unit tests constructing `Config(priority=...)` with enum, valid strings in mixed case, and an unknown string expecting `pydantic.ValidationError` (assert with `pytest.raises(..., match=...)`). + +## Open questions + +None. The wire contract, resolution rule, transport coverage, and testing approach are all fixed by the PRD and confirmed against the current code. diff --git a/dev/specs/ihs-259-sdk-x-priority-header/spec.md b/dev/specs/ihs-259-sdk-x-priority-header/spec.md new file mode 100644 index 000000000..108bd61fb --- /dev/null +++ b/dev/specs/ihs-259-sdk-x-priority-header/spec.md @@ -0,0 +1,156 @@ +# Feature Specification: SDK `X-Priority` Request Header + +**Feature Branch**: `dga/feat-x-priority-aa2nd` + +**Created**: 2026-07-10 + +**Status**: Draft + +**Input**: Jira IHS-259 — "feat: SDK X-Priority request header" (PRD). Related: INFP-636 (server-side prioritization, parent), GitHub #1151 (SDK feature issue), GitHub #1124 (429 retry/backoff, complementary). + +## Overview + +Infrahub's background systems (generators, artifacts, diffs, syncs, computed attributes) call back into the same API servers through this SDK, competing on equal footing with human/frontend traffic for a shared worker pool and database connections. Today the API layer cannot distinguish interactive from background traffic, so under heavy background load it cannot preferentially protect the frontend. + +This feature gives the SDK a first-class notion of request **priority** (`high | normal | low`) that it emits as an `X-Priority` HTTP header. The originator of each request — the SDK caller — declares how important the request is, in a form the server can act on. Priority is set two ways: a **client-wide default** (a client dedicated to background work tags everything `low` with no call-site changes) and a **per-request override** on individual operations. When nothing is configured, the SDK sends no header and behaves exactly as today; the server treats absent/unknown values as `normal`, so rollout is safe and incremental. + +This is the SDK's contribution to the server-side prioritization effort in INFP-636. It does **not** implement server-side admission control or 429 handling. + +## User Scenarios & Testing *(mandatory)* + +### User Story 1 - Client-wide default priority (Priority: P1) + +An operator running background workloads constructs one client with a default priority. Every request that client issues then carries `X-Priority: ` with no changes at any call site, so the server can shed that traffic before frontend traffic. + +**Why this priority**: This is the core value of the feature — it lets an entire background workload be tagged uniformly by configuration alone, which is the dominant real-world use case (generators, syncs, `infrahubctl`, the Ansible collection). It is the minimum viable slice: shipping only this already lets operators protect frontend traffic. + +**Independent Test**: Construct a client with `priority=low`, issue one request of each transport type (GraphQL query/mutation, multipart file upload, raw blob `_get`/`_post`), and assert every outgoing request carries `X-Priority: low`. Fully testable without the per-request override existing. + +**Acceptance Scenarios**: + +1. **Given** a client built with a default priority of `low`, **When** it issues a GraphQL query or mutation, **Then** that request carries `X-Priority: low`. +2. **Given** a client built with a default priority of `low`, **When** it issues a multipart file upload, **Then** that request carries `X-Priority: low`. +3. **Given** a client built with a default priority of `low`, **When** it issues a raw blob transfer (`_get`/`_post`), **Then** that request carries `X-Priority: low`. +4. **Given** a client built with a default priority of `normal`, **When** it issues any request, **Then** that request carries `X-Priority: normal` (an explicitly configured default is always emitted). + +--- + +### User Story 2 - Per-request override (Priority: P2) + +A caller overrides priority for a single operation on an otherwise-default client — for example, tagging one user-triggered operation `high` on a client that is otherwise dedicated to background work — without affecting any other call. + +**Why this priority**: Adds targeted control on top of the client-wide default. Valuable but secondary: the default alone delivers the primary outcome, and the override is only meaningful once defaults exist. + +**Independent Test**: On a client with no configured default, invoke a covered method with `priority=Priority.HIGH`, assert that one request carries `X-Priority: high`, then invoke the same method with no priority argument and assert no `X-Priority` header is present. + +**Acceptance Scenarios**: + +1. **Given** a client with no configured default, **When** the caller invokes a covered method with `priority=Priority.HIGH`, **Then** that one request carries `X-Priority: high`. +2. **Given** a client with no configured default, **When** the caller invokes a covered method with no priority argument, **Then** that request carries no `X-Priority` header, and a prior override does not leak into it. +3. **Given** a client with a default priority of `low`, **When** the caller invokes a covered method with `priority=Priority.HIGH`, **Then** that one request carries `X-Priority: high` and the next un-annotated call reverts to `X-Priority: low`. + +--- + +### User Story 3 - Zero behaviour change when unconfigured (Priority: P1) + +An existing SDK user who sets no priority sees no change to outgoing requests after upgrading the SDK. No `X-Priority` header is added anywhere. + +**Why this priority**: Backwards compatibility is a hard safety requirement for a foundational library. It is P1 because a regression here silently changes every existing user's traffic. It is independently testable and gates safe rollout. + +**Independent Test**: With a client constructed exactly as before this feature (no priority configured, no per-request argument), assert that outgoing requests across all transports contain no `X-Priority` header — identical to pre-feature behaviour. + +**Acceptance Scenarios**: + +1. **Given** a client with no priority configured, **When** it issues any request across any transport, **Then** no `X-Priority` header is present. +2. **Given** a client with no priority configured and a call that passes no priority argument, **When** the request is issued, **Then** the outgoing request headers are identical to pre-feature behaviour. + +--- + +### User Story 4 - Invalid configured priority rejected loudly (Priority: P2) + +An SDK developer or operator who configures an invalid priority value (a typo such as `lowe`, or any value outside the closed set) gets a loud failure at configuration-load time, not a silent malformed header on the wire. + +**Why this priority**: Fail-fast on misconfiguration prevents silently shedding or over-prioritising traffic. Secondary to the happy paths but important for operational trust. + +**Independent Test**: Attempt to construct configuration with an unknown priority value and assert it raises a configuration/validation error before any request is issued. + +**Acceptance Scenarios**: + +1. **Given** configuration with a priority value outside the closed set (e.g. `lowe`), **When** the configuration is loaded, **Then** loading fails with a validation error and no client is created. +2. **Given** configuration with a valid priority value in any letter case (e.g. `LOW`, `Low`, `low`), **When** the configuration is loaded, **Then** it is accepted and normalised to the corresponding priority. + +--- + +### User Story 5 - Async and sync parity (Priority: P1) + +A developer using the synchronous client (`InfrahubClientSync`) gets behaviour identical to the asynchronous client (`InfrahubClient`) for every aspect of this feature — configuration, defaults, per-request override, resolution, and the unconfigured no-header case. + +**Why this priority**: The dual async/sync pattern is mandatory in this SDK (per AGENTS.md). Divergence would make the choice of client style silently change semantics. P1 because it is a correctness invariant across the whole feature rather than an add-on. + +**Independent Test**: Run the identical assertion suite (defaults, override, omit-vs-emit, resolution) against both clients and assert identical outcomes. + +**Acceptance Scenarios**: + +1. **Given** the same priority configuration, **When** the assertion suite runs against `InfrahubClient` and against `InfrahubClientSync`, **Then** both produce identical outgoing `X-Priority` behaviour. + +--- + +### Edge Cases + +- **Explicit step-up on a low-default client**: a per-request `NORMAL` on a client whose default is `low` sends `X-Priority: normal` for that call — explicit intent wins, even when stepping *up* from the default. +- **No "send no header" per-request escape once a default is set**: once a client default is configured there is no per-request way to suppress the header; the accepted equivalent is passing `NORMAL` explicitly. +- **Batch mode and raw blob transfers**: these inherit the client default but expose **no** per-call override in v1. +- **Invalid configured value**: errors at configuration load, never at request time. +- **Caller manually pre-populates an `X-Priority` header**: only reachable via the low-level `_get`/`_post` transport methods, which accept a raw `headers=` argument (the covered public methods do not). The resolution rule is the single source of truth (documented behaviour); manually injecting the header at that low level is not a supported side channel and its interaction with resolution is not guaranteed. +- **Explicit per-request `NORMAL` vs. no argument**: an explicit `NORMAL` always emits `X-Priority: normal`; a `None`/absent per-request value falls through to the client default (which may itself be absent, in which case no header is sent). + +## Requirements *(mandatory)* + +### Functional Requirements + +- **FR-001**: System MUST expose a closed set of priority values `high | normal | low` as a `Priority` enum, so callers express intent as a typed value rather than a raw header string. +- **FR-002**: Users MUST be able to configure a client-wide default priority via configuration, accepting either a `Priority` enum value or a case-insensitive string (through environment or file configuration). +- **FR-003**: When a default priority is configured, System MUST attach the `X-Priority` header to **every** outgoing request across all transports — GraphQL queries/mutations, multipart uploads, and raw blob `_get`/`_post`. +- **FR-004**: When no priority is configured and none is supplied per request, System MUST omit the `X-Priority` header entirely, producing outgoing requests byte-for-byte identical to current (pre-feature) behaviour. +- **FR-005**: Users MUST be able to override priority per request via a `priority` argument (accepting a `Priority` value or `None`, default `None`) on the covered public methods: `get`, `all`, `create`, `save`, the diff methods, `execute_graphql`, and its file variant. +- **FR-006**: Priority resolution MUST be `resolved = per_request if per_request is not None else client_default`. A resolved value of `None` MUST omit the header; a resolved explicit value MUST be sent, including an explicit `NORMAL`, which MUST send `X-Priority: normal`. +- **FR-007**: System MUST reject an invalid or unknown configured priority value at configuration-load time (a validation/type error) rather than coercing it or silently sending it. +- **FR-008**: The asynchronous client (`InfrahubClient`) and the synchronous client (`InfrahubClientSync`) MUST behave identically for every aspect of this feature. +- **FR-009**: The emitted header MUST be named exactly `X-Priority`, carrying the lowercase value string (`high`, `normal`, `low`) that corresponds to the resolved priority. + +### Key Entities *(include if feature involves data)* + +- **Priority** *(new)*: a closed enumeration with members `HIGH`, `NORMAL`, `LOW`. It is the single in-code representation of request priority, owned by the SDK, with no lifecycle beyond its value. Each member maps to a lowercase wire value (`high`/`normal`/`low`). +- **Configuration** *(extended)*: gains a `priority` field defaulting to `None` (meaning "no default; omit the header"). Accepts a `Priority` value or a case-insensitive string, validated at load time. +- **Base request headers** *(extended)*: the client's base header set is extended so a configured default is injected once and rides every transport, rather than being added at each call site. + +## Success Criteria *(mandatory)* + +### Measurable Outcomes + +- **SC-001**: A client with a configured default emits `X-Priority: ` on 100% of requests, verified across GraphQL, multipart upload, and blob `_get`/`_post`. +- **SC-002**: A client with no priority configured emits no `X-Priority` header, and no other SDK-set outgoing header changes versus current behaviour (asserted by `X-Priority` being absent from the captured request; not a literal byte-for-byte comparison of transport-injected headers). +- **SC-003**: A per-request override sets the header on exactly that request and leaves the client default intact for the next call. +- **SC-004**: An invalid priority value is rejected at configuration load rather than sent or coerced. +- **SC-005**: The async and sync clients pass the identical assertion suite with identical outcomes. +- **SC-006**: A client with a configured default emits `X-Priority: ` on requests issued via batch mode and via raw blob transfers, confirming those transports inherit the client default (even though they expose no per-request override in v1). + +## Assumptions + +- **Server contract (per INFP-636)**: the header is exactly `X-Priority`; values are case-insensitive on the server; absent or unknown values are treated as `normal` server-side. This makes an absent header and a `normal` value semantically equivalent to the server, which is why omitting the header when unconfigured is safe. +- **Dual async/sync pattern is mandatory**: every change here is applied to both `InfrahubClient` and `InfrahubClientSync`, per AGENTS.md. +- **Public API signature change is accepted**: this feature adds a new enum, a new configuration field, and a new `priority` keyword argument across the covered public method surface — an intentional public-API-signature change (flagged per AGENTS.md "ask first: changing public API signatures"). +- **Docs regeneration is required**: the new configuration field and docstrings require `uv run invoke docs-generate`. +- **Prior art guides implementation**: existing header handling in the client (notably `X-Infrahub-Tracker`) and current `tests/unit` client request tests are the reference for how headers are injected and asserted. + +## Out of Scope + +- 429 / `Retry-After` retry and backoff handling → tracked in GitHub #1124 (complementary, separate). +- Server-side admission control, dedicated-capacity routing, or database-throttle priority awareness → server side, tracked under INFP-636. +- Traffic classification guidance or auto-tagging of call sites (deciding *which* priority a given workload should use). +- Per-batch or per-blob override knobs (batch and blob transfers inherit the client default only, in v1). +- Anti-escalation enforcement (a usage guideline, not enforced in code). + +## Dependencies + +- **INFP-636** — server-side API Request Prioritization (parent effort that defines and consumes the `X-Priority` contract). This SDK feature is only useful once the server acts on the header, but is safe to ship independently because absent/unknown is treated as `normal`. diff --git a/dev/specs/ihs-259-sdk-x-priority-header/tasks.md b/dev/specs/ihs-259-sdk-x-priority-header/tasks.md new file mode 100644 index 000000000..c0ad370ca --- /dev/null +++ b/dev/specs/ihs-259-sdk-x-priority-header/tasks.md @@ -0,0 +1,200 @@ +--- +description: "Task list for SDK X-Priority request header (IHS-259)" +--- + +# Tasks: SDK `X-Priority` Request Header + +**Input**: Design documents from `specs/ihs-259-sdk-x-priority-header/` + +**Prerequisites**: plan.md, spec.md, research.md, data-model.md, contracts/, quickstart.md + +**Tests**: Included — the PRD (IHS-259 "Testing Decisions") and Success Criteria SC-001…SC-006 explicitly require unit + contract tests. Tests are first-class here. + +**Organization**: Tasks are grouped by user story (from spec.md), in priority order. The MVP is User Story 1. + +## Format: `[ID] [P?] [Story] Description` + +- **[P]**: Can run in parallel (different files, no dependency on an incomplete task) +- **[Story]**: US1…US5 (setup / foundational / polish tasks have no story label) + +## Path Conventions + +Single-project Python library. Production code under `infrahub_sdk/`; tests under `tests/unit/sdk/`. Paths below are repository-relative (the `specs/` symlink resolves to `dev/specs/`). + +## Prior art to mirror + +- Header injection & per-request merge: `X-INFRAHUB-KEY` (`infrahub_sdk/client.py:217-218`) and `X-Infrahub-Tracker` (`client.py:1244-1246`, `1329-1333`, `2225-2226`, `2312-2313`). +- Enum pattern: `InfrahubClientMode(str, enum.Enum)` (`infrahub_sdk/constants.py:4`). +- Config enum field: `mode` / `transport` (`config.py:57`, `config.py:87`). +- Header-on-the-wire tests: `match_headers={...}` (`tests/unit/sdk/test_object_store.py:22-29`, `test_client.py:366+`); `BothClients` parity fixture (`tests/unit/sdk/conftest.py:33-45`). + +--- + +## Phase 1: Setup + +**Purpose**: Confirm the working environment before touching code. + +- [X] T001 Ensure dev dependencies are installed and the baseline is green: run `uv sync --all-groups --all-extras` then `uv run pytest tests/unit/sdk/test_config.py tests/unit/sdk/test_client.py -q` to confirm a clean starting point. (Done: 107 passed on baseline.) + +--- + +## Phase 2: Foundational (Blocking Prerequisites) + +**Purpose**: The `Priority` enum and `Config.priority` field are prerequisites for every user story. MUST complete before Phase 3+. + +- [X] T002 Add `class Priority(str, enum.Enum)` with members `HIGH = "high"`, `NORMAL = "normal"`, `LOW = "low"` and a case-insensitive `_missing_` classmethod (returns the member matching `value.lower()`, else `None`) to `infrahub_sdk/constants.py` (mirror `InfrahubClientMode`). Add module docstring/type hints per repo style. +- [X] T003 [P] Export `Priority` from the SDK public namespace: add it to `infrahub_sdk/__init__.py` imports and `__all__` (alongside other public enums), per contracts/priority-api.md. +- [X] T004 Add `priority: Priority | None = Field(default=None, description="Default request priority emitted as the X-Priority header on every request; one of high|normal|low (case-insensitive). When unset, no header is sent.")` to `ConfigBase` in `infrahub_sdk/config.py`; import `Priority` from `.constants`. Confirm the field auto-binds to `INFRAHUB_PRIORITY` (no custom source needed) and is carried by `Config.clone()` (it iterates `model_fields`, so no change to `clone()` required — verify only). + +**Checkpoint**: `Priority` importable from `infrahub_sdk`; `Config(priority=...)` accepts enum/string; the SDK still imports and existing tests still pass. + +--- + +## Phase 3: User Story 1 - Client-wide default priority (Priority: P1) 🎯 MVP + +**Goal**: A client built with a default priority emits `X-Priority: ` on every request across all transports, with no call-site changes. + +**Independent test**: Construct a client with `priority=Priority.LOW`; issue one GraphQL, one multipart upload, and one blob `_get`/`_post`; assert each outgoing request carries `X-Priority: low`. + +### Implementation + +- [X] T005 [US1] In `BaseClient.__init__` (`infrahub_sdk/client.py`, next to the `X-INFRAHUB-KEY` block ~line 217-218), inject the default once: `if self.config.priority is not None: self.headers["X-Priority"] = self.config.priority.value`. This single edit covers async and sync (both subclass `BaseClient`) and, because every transport re-merges `self.headers`, rides GraphQL, multipart, and raw blob transports automatically. + +### Tests + +- [X] T006 [P] [US1] Add `tests/unit/sdk/test_priority.py`: assert a `priority=Priority.LOW` client emits `X-Priority: low` on a GraphQL query and a mutation, for both clients (`match_headers={"X-Priority": "low"}`, parametrized over the `BothClients` fixture). (SC-001, FR-003) +- [X] T007 [P] [US1] In `tests/unit/sdk/test_object_store.py` (or `test_priority.py`), assert a `priority=Priority.LOW` client emits `X-Priority: low` on a blob download (`_get_streaming`) and upload (`_post`/object-store), both clients. (SC-001, SC-006 blob) +- [X] T008 [P] [US1] Add a multipart-upload test: a `priority=Priority.LOW` client emits `X-Priority: low` on `_execute_graphql_with_file`, both clients (confirm the header survives the `content-type` pop). (SC-001, FR-003) +- [X] T009 [P] [US1] Add a batch-mode test: a `priority=Priority.LOW` client issues a batched operation and every batched request carries `X-Priority: low`. (SC-006 batch) +- [X] T010 [P] [US1] Add a test that a `priority=Priority.NORMAL` client emits `X-Priority: normal` on requests (an explicitly configured default is always emitted, not omitted), both clients. (US1 acceptance #4, FR-006) + +**Checkpoint**: US1 fully testable and green independently — this is a shippable MVP. + +--- + +## Phase 4: User Story 3 - Zero behaviour change when unconfigured (Priority: P1) + +**Goal**: A client with no priority configured (and no per-request arg) emits no `X-Priority` header anywhere — identical to the pre-feature SDK. + +**Independent test**: Construct a client with no `priority`; issue requests across transports; assert no `X-Priority` header is present. + +**Note**: The conditional in T005 already implements the omit path; this phase locks it with tests. Depends on T005. + +### Tests + +- [X] T011 [P] [US3] In `tests/unit/sdk/test_priority.py`, assert an unconfigured client emits **no** `X-Priority` header across GraphQL, multipart, and blob transports — capture the request via `httpx_mock.get_requests()` and assert `"x-priority" not in request.headers`, both clients. (SC-002, FR-004) +- [X] T012 [P] [US3] Assert that with no priority configured, no per-request arg, the SDK-set outgoing headers are unchanged versus baseline (only `X-Priority` absence matters; do not assert on transport-injected headers like host/user-agent). (SC-002) + +**Checkpoint**: Backwards compatibility proven for both clients. + +--- + +## Phase 5: User Story 2 - Per-request override (Priority: P2) + +**Goal**: A `priority=` argument on the covered public methods overrides the client default for exactly one request, resolving as `per_request if per_request is not None else client_default`. + +**Independent test**: On a client with no default, call a covered method with `priority=Priority.HIGH` (asserts `X-Priority: high`), then call it again with no arg (asserts no header). + +### Implementation + +- [X] T013 [US2] Add `priority: Priority | None = None` to async `execute_graphql` (`client.py:1201`) and apply the override after the existing `headers = copy.copy(self.headers or {})` + tracker block: `if priority is not None: headers["X-Priority"] = priority.value`. (FR-005, FR-006) +- [X] T014 [US2] Add `priority: Priority | None = None` to async `_execute_graphql_with_file` (`client.py:1290`); apply the override **after** the copy and the `content-type` pop so it is not lost. (FR-005, FR-006, critique E3) +- [X] T015 [US2] Mirror T013–T014 on the sync client: `execute_graphql` (`client.py:2181`) and `_execute_graphql_with_file` (`client.py:2270`). (FR-008) +- [X] T016 [US2] Thread `priority` through the client high-level methods so they forward it to the execute funnels: async `get` (`client.py:442`), `all`→`filters` (`client.py:905`/`1131`), `create` (`client.py:400`); ensure the pagination loop in `filters` forwards `priority` on **every** page request. (FR-005, critique E2) +- [X] T017 [US2] Thread `priority` through the diff methods: `create_diff` (`client.py:1695`), `get_diff_summary` (`client.py:1724`), `get_diff_tree` (`client.py:1763`), forwarding to `execute_graphql`. (FR-005) +- [X] T018 [US2] Mirror T016–T017 on the sync client (`get` `client.py:2975`, `all` `client.py:2639`/`2907`, `create` `client.py:2137`, `create_diff` `client.py:3266`, `get_diff_summary`/`get_diff_tree`). (FR-008) +- [X] T019 [US2] Add `priority: Priority | None = None` to async node methods and forward to the client execute calls: `save` (`node/node.py:1241`), `create` (`node/node.py:1602`), `update` (`node/node.py:1681`), `delete` (`node/node.py:1214`). (FR-005) +- [X] T020 [US2] Mirror T019 on the sync node (`InfrahubNodeSync`: `delete` `node/node.py:2402`, `save` `node/node.py:2429`, plus `create`/`update`). (FR-008) + +### Tests + +- [X] T021 [P] [US2] Test: no-default client + `priority=Priority.HIGH` on `execute_graphql` emits `X-Priority: high`; a following un-annotated call emits no header (no leak). Both clients. (SC-003, US2 acceptance #1/#2) +- [X] T022 [P] [US2] Test: `priority=Priority.LOW` default client + per-request `priority=Priority.HIGH` emits `X-Priority: high` for that call, and the next un-annotated call reverts to `X-Priority: low`. Both clients. (SC-003, US2 acceptance #3) +- [X] T023 [P] [US2] Test: `priority=Priority.LOW` default client + per-request `priority=Priority.NORMAL` emits `X-Priority: normal` (explicit step-up wins). Both clients. (spec Edge Cases, SC-003) +- [X] T024 [P] [US2] Test the override on the covered surfaces: `get`, `all` (multi-page — assert every page request carries the override), `create`, `save`, a diff method, and `_execute_graphql_with_file`. Both clients. (FR-005, critique E2/E3) + +**Checkpoint**: Override works and resolves correctly on every covered surface, both clients. + +--- + +## Phase 6: User Story 4 - Invalid configured priority rejected (Priority: P2) + +**Goal**: An invalid/unknown configured priority fails at config load; valid strings in any case are accepted. + +**Independent test**: `Config(priority="lowe")` raises; `Config(priority="LOW").priority is Priority.LOW`. + +**Note**: Validation comes for free from the enum + `_missing_` (T002/T004); this phase locks it with tests. + +### Tests + +- [X] T025 [P] [US4] In `tests/unit/sdk/test_config.py`, assert `Config(address="http://localhost:8000", priority="lowe")` raises `pydantic.ValidationError` (use `pytest.raises(..., match=...)`); assert no request is issued. (SC-004, FR-007) +- [X] T026 [P] [US4] Assert case-insensitive acceptance: `Config(priority="LOW")`, `Config(priority="Low")`, `Config(priority="low")`, and `Config(priority=Priority.LOW)` all yield `Priority.LOW`; likewise for HIGH/NORMAL. Include the env-var path `INFRAHUB_PRIORITY=LOW` via `monkeypatch`. (SC-004, FR-002) +- [X] T027 [P] [US4] Assert `Config()` default → `priority is None` (no default, header omitted). (FR-004) + +**Checkpoint**: Misconfiguration fails loudly; valid config in any case is accepted. + +--- + +## Phase 7: User Story 5 - Async / sync parity (Priority: P1) + +**Goal**: Every aspect behaves identically on `InfrahubClient` and `InfrahubClientSync`. + +**Independent test**: The same assertion suite runs against both clients with identical outcomes. + +- [X] T028 [US5] Audit T006–T012 and T021–T024 to confirm every wire/resolution test is parametrized over the `BothClients` fixture (`["standard","sync"]`); add parametrization to any that isn't. (SC-005, FR-008) +- [X] T029 [P] [US5] Add a focused parity test asserting the resolution truth table (data-model.md) produces identical emitted headers for both clients across the default × override combinations. (SC-005) + +**Checkpoint**: Parity is explicit and enforced, not incidental. + +--- + +## Phase 8: Polish & Cross-Cutting Concerns + +**Purpose**: Docs, quality gates, and release hygiene. + +- [X] T030 Add docstrings to the new `Priority` enum, the `Config.priority` field, and the `priority` kwarg on the covered public methods (drives generated docs). +- [X] T031 Run `uv run invoke docs-generate`, then `uv run invoke docs-validate`; commit the regenerated docs (new `Config.priority` field). (Governance gate: docs regeneration) +- [X] T032 [P] Add a changelog fragment under `changelog/` (mirror the existing fragment style, e.g. an `.added.md` for the new `Priority`/`Config.priority`/`priority=` surface referencing IHS-259 / #1151). +- [X] T033 Run `uv run invoke format lint-code` (ruff, ty, mypy) and fix any findings; confirm type hints on all new/changed signatures. +- [X] T034 Run the full `uv run pytest tests/unit/` suite and confirm green (including all new priority tests for both clients). +- [X] T035 Validate against quickstart.md: run the mapped validation scenarios and confirm SC-001…SC-006 are all covered. + +--- + +## Dependencies & Execution Order + +- **Phase 1 (Setup)** → **Phase 2 (Foundational: T002–T004)** must complete first; they block everything. +- **Phase 3 (US1)** depends on T004 (config field) + T005 (base injection). This is the MVP. +- **Phase 4 (US3)** depends on T005 (shares the conditional-injection code). Can be developed right after US1. +- **Phase 5 (US2)** depends on Foundational (enum) and is independent of US1's base injection for its core (works on a no-default client), but its "override beats default" tests (T022–T023) depend on T005. +- **Phase 6 (US4)** depends only on Foundational (T002/T004). +- **Phase 7 (US5)** depends on the tests from US1–US4 existing (it audits/extends them). +- **Phase 8 (Polish)** last. + +### Story independence + +- US1, US3, US4 are each independently testable after Foundational. +- US2 is independently testable on a no-default client after Foundational; full override-vs-default coverage wants T005. +- US5 is a cross-cutting invariant realized by parametrizing the other stories' tests. + +### Parallelization + +- **Foundational**: T003 [P] can run alongside T002/T004 once `Priority` exists. +- **Within US1**: T006–T010 are all [P] (independent test files/cases) once T005 lands. +- **Within US2**: implementation T013–T020 touch overlapping regions of `client.py`/`node.py` (mostly sequential per file); tests T021–T024 are [P]. +- **Within US4**: T025–T027 are [P]. +- **Polish**: T032 [P]; T031/T033/T034 are sequential gates. + +## Implementation Strategy + +1. **MVP first**: Phases 1–3 (Setup → Foundational → US1). Ship a client-wide default that rides every transport. +2. **Lock safety**: Phase 4 (US3) — prove zero behaviour change when unconfigured. +3. **Add control**: Phase 5 (US2) — per-request override. +4. **Harden**: Phases 6–7 (US4 validation, US5 parity). +5. **Finish**: Phase 8 — docs, changelog, quality gates. + +## Task summary + +- **Total tasks**: 35 +- **By story**: Foundational 3 (T002–T004) + Setup 1 + US1 6 (T005–T010) + US3 2 (T011–T012) + US2 12 (T013–T024) + US4 3 (T025–T027) + US5 2 (T028–T029) + Polish 6 (T030–T035) +- **MVP scope**: T001–T010 (Setup + Foundational + US1) diff --git a/docs/docs/infrahubctl/infrahubctl-schema.mdx b/docs/docs/infrahubctl/infrahubctl-schema.mdx index 34f46844a..ec9f24fe3 100644 --- a/docs/docs/infrahubctl/infrahubctl-schema.mdx +++ b/docs/docs/infrahubctl/infrahubctl-schema.mdx @@ -21,6 +21,7 @@ $ infrahubctl schema [OPTIONS] COMMAND [ARGS]... * `export`: Export the schema from Infrahub as YAML... * `list`: List all available schema kinds. * `show`: Show details for a specific schema kind. +* `format`: Format Infrahub schema files with a... ## `infrahubctl schema load` @@ -133,3 +134,45 @@ $ infrahubctl schema show [OPTIONS] KIND * `-b, --branch TEXT`: Target branch * `--config-file TEXT`: [env var: INFRAHUBCTL_CONFIG; default: infrahubctl.toml] * `--help`: Show this message and exit. + +## `infrahubctl schema format` + +Format Infrahub schema files with a canonical key ordering. + +Reorders the keys within each node, generic, attribute, relationship and +dropdown choice into a consistent, opinionated order so schema files read +the same way and produce small diffs. + +Only your own nodes are formatted; nodes in Infrahub-reserved namespaces are +left untouched. Comments, quoting, and inline (flow) sequences are preserved. + +By default the change is purely key ordering. The opt-in flags additionally +change content: --strip-defaults drops redundant default values, +--sort-by-order-weight reorders attributes/relationships, and +--backfill-order-weight fills in a missing order_weight. + +Examples: + infrahubctl schema format schemas/ + infrahubctl schema format schemas/dcim.yml --diff + infrahubctl schema format schemas/ --check + infrahubctl schema format schemas/ --strip-defaults --sort-by-order-weight + +**Usage**: + +```console +$ infrahubctl schema format [OPTIONS] SCHEMAS... +``` + +**Arguments**: + +* `SCHEMAS...`: [required] + +**Options**: + +* `--check`: Do not write files; exit 1 if any file would be reformatted. +* `--diff`: Print a diff of the changes instead of writing files. +* `--strip-defaults`: Remove attribute/relationship/node keys whose value equals the schema default. +* `--sort-by-order-weight`: Sort attributes and relationships by order_weight (items without one keep their order and go last). +* `--backfill-order-weight`: Give attributes/relationships that lack an order_weight the value 1000. +* `--config-file TEXT`: [env var: INFRAHUBCTL_CONFIG; default: infrahubctl.toml] +* `--help`: Show this message and exit. diff --git a/docs/docs/python-sdk/guides/branches.mdx b/docs/docs/python-sdk/guides/branches.mdx index 64016048e..ffe2049d7 100644 --- a/docs/docs/python-sdk/guides/branches.mdx +++ b/docs/docs/python-sdk/guides/branches.mdx @@ -108,15 +108,40 @@ The Python SDK provides multiple methods to manage the branches in an Infrahub i -## Generating a diff for a branch +## Getting the diff for a branch + +Use `get_diff_tree` to retrieve the full diff of a branch compared to its base branch, including summary counts and the list of changed nodes. It returns `None` if no diff exists for the branch. + + + + + ```python + from infrahub_sdk import InfrahubClient + client = InfrahubClient() + diff = await client.get_diff_tree(branch="new-branch") + ``` + + + + + ```python + from infrahub_sdk import InfrahubClientSync + client = InfrahubClientSync() + diff = client.get_diff_tree(branch="new-branch") + ``` + + + + +If you only need the list of changed nodes, `get_diff_summary` returns them without the diff metadata. ```python from infrahub_sdk import InfrahubClient - client = await InfrahubClient() - diff = await client.branch.diff_data(branch_name="new-branch") + client = InfrahubClient() + node_diffs = await client.get_diff_summary(branch="new-branch") ``` @@ -125,7 +150,7 @@ The Python SDK provides multiple methods to manage the branches in an Infrahub i ```python from infrahub_sdk import InfrahubClientSync client = InfrahubClientSync() - diff = client.branch.diff_data(branch_name="new-branch") + node_diffs = client.get_diff_summary(branch="new-branch") ``` diff --git a/docs/docs/python-sdk/reference/config.mdx b/docs/docs/python-sdk/reference/config.mdx index ffd36fd6a..b8750fe0a 100644 --- a/docs/docs/python-sdk/reference/config.mdx +++ b/docs/docs/python-sdk/reference/config.mdx @@ -133,6 +133,14 @@ The following settings can be defined in the `Config` class **Environment variable**: `INFRAHUB_PAGINATION_SIZE`
+## priority + + +**Description**: Default request priority emitted as the X-Priority header on every request; one of high|medium|low (case-insensitive). When unset, no header is sent.
+**Type**: `object`
+**Environment variable**: `INFRAHUB_PRIORITY`
+ + ## retry_delay @@ -151,6 +159,42 @@ The following settings can be defined in the `Config` class **Environment variable**: `INFRAHUB_RETRY_ON_FAILURE`
+## rate_limit_retry_enabled + + +**Description**: Retry requests that receive HTTP 429 using backoff. Set False to disable.
+**Type**: `boolean`
+**Default value**: True
+**Environment variable**: `INFRAHUB_RATE_LIMIT_RETRY_ENABLED`
+ + +## rate_limit_max_retries + + +**Description**: Maximum number of retries after the initial attempt when receiving HTTP 429.
+**Type**: `integer`
+**Default value**: 10
+**Environment variable**: `INFRAHUB_RATE_LIMIT_MAX_RETRIES`
+ + +## rate_limit_backoff_base + + +**Description**: Base interval in seconds for exponential backoff between 429 retries.
+**Type**: `number`
+**Default value**: 0.5
+**Environment variable**: `INFRAHUB_RATE_LIMIT_BACKOFF_BASE`
+ + +## rate_limit_backoff_max + + +**Description**: Maximum wait in seconds for any single 429 retry (also clamps Retry-After).
+**Type**: `number`
+**Default value**: 60.0
+**Environment variable**: `INFRAHUB_RATE_LIMIT_BACKOFF_MAX`
+ + ## max_retry_duration diff --git a/docs/docs/python-sdk/sdk_ref/infrahub_sdk/client.mdx b/docs/docs/python-sdk/sdk_ref/infrahub_sdk/client.mdx index e858f2179..1f783d46c 100644 --- a/docs/docs/python-sdk/sdk_ref/infrahub_sdk/client.mdx +++ b/docs/docs/python-sdk/sdk_ref/infrahub_sdk/client.mdx @@ -16,7 +16,7 @@ GraphQL Client to interact with Infrahub. #### `get` ```python -get(self, kind: type[SchemaType], raise_when_missing: Literal[False], at: Timestamp | None = ..., branch: str | None = ..., timeout: int | None = ..., id: str | None = ..., hfid: list[str] | None = ..., include: list[str] | None = ..., exclude: list[str] | None = ..., populate_store: bool = ..., fragment: bool = ..., prefetch_relationships: bool = ..., property: bool = ..., include_metadata: bool = ..., query_name: str | None = ..., **kwargs: Any) -> SchemaType | None +get(self, kind: type[SchemaType], raise_when_missing: Literal[False], at: Timestamp | None = ..., branch: str | None = ..., timeout: int | None = ..., id: str | None = ..., hfid: list[str] | None = ..., include: list[str] | None = ..., exclude: list[str] | None = ..., populate_store: bool = ..., fragment: bool = ..., prefetch_relationships: bool = ..., property: bool = ..., include_metadata: bool = ..., query_name: str | None = ..., priority: Priority | None = ..., **kwargs: Any) -> SchemaType | None ```
@@ -25,37 +25,37 @@ get(self, kind: type[SchemaType], raise_when_missing: Literal[False], at: Timest #### `get` ```python -get(self, kind: type[SchemaType], raise_when_missing: Literal[True], at: Timestamp | None = ..., branch: str | None = ..., timeout: int | None = ..., id: str | None = ..., hfid: list[str] | None = ..., include: list[str] | None = ..., exclude: list[str] | None = ..., populate_store: bool = ..., fragment: bool = ..., prefetch_relationships: bool = ..., property: bool = ..., include_metadata: bool = ..., query_name: str | None = ..., **kwargs: Any) -> SchemaType +get(self, kind: type[SchemaType], raise_when_missing: Literal[True], at: Timestamp | None = ..., branch: str | None = ..., timeout: int | None = ..., id: str | None = ..., hfid: list[str] | None = ..., include: list[str] | None = ..., exclude: list[str] | None = ..., populate_store: bool = ..., fragment: bool = ..., prefetch_relationships: bool = ..., property: bool = ..., include_metadata: bool = ..., query_name: str | None = ..., priority: Priority | None = ..., **kwargs: Any) -> SchemaType ``` #### `get` ```python -get(self, kind: type[SchemaType], raise_when_missing: bool = ..., at: Timestamp | None = ..., branch: str | None = ..., timeout: int | None = ..., id: str | None = ..., hfid: list[str] | None = ..., include: list[str] | None = ..., exclude: list[str] | None = ..., populate_store: bool = ..., fragment: bool = ..., prefetch_relationships: bool = ..., property: bool = ..., include_metadata: bool = ..., query_name: str | None = ..., **kwargs: Any) -> SchemaType +get(self, kind: type[SchemaType], raise_when_missing: bool = ..., at: Timestamp | None = ..., branch: str | None = ..., timeout: int | None = ..., id: str | None = ..., hfid: list[str] | None = ..., include: list[str] | None = ..., exclude: list[str] | None = ..., populate_store: bool = ..., fragment: bool = ..., prefetch_relationships: bool = ..., property: bool = ..., include_metadata: bool = ..., query_name: str | None = ..., priority: Priority | None = ..., **kwargs: Any) -> SchemaType ``` #### `get` ```python -get(self, kind: str, raise_when_missing: Literal[False], at: Timestamp | None = ..., branch: str | None = ..., timeout: int | None = ..., id: str | None = ..., hfid: list[str] | None = ..., include: list[str] | None = ..., exclude: list[str] | None = ..., populate_store: bool = ..., fragment: bool = ..., prefetch_relationships: bool = ..., property: bool = ..., include_metadata: bool = ..., query_name: str | None = ..., **kwargs: Any) -> InfrahubNode | None +get(self, kind: str, raise_when_missing: Literal[False], at: Timestamp | None = ..., branch: str | None = ..., timeout: int | None = ..., id: str | None = ..., hfid: list[str] | None = ..., include: list[str] | None = ..., exclude: list[str] | None = ..., populate_store: bool = ..., fragment: bool = ..., prefetch_relationships: bool = ..., property: bool = ..., include_metadata: bool = ..., query_name: str | None = ..., priority: Priority | None = ..., **kwargs: Any) -> InfrahubNode | None ``` #### `get` ```python -get(self, kind: str, raise_when_missing: Literal[True], at: Timestamp | None = ..., branch: str | None = ..., timeout: int | None = ..., id: str | None = ..., hfid: list[str] | None = ..., include: list[str] | None = ..., exclude: list[str] | None = ..., populate_store: bool = ..., fragment: bool = ..., prefetch_relationships: bool = ..., property: bool = ..., include_metadata: bool = ..., query_name: str | None = ..., **kwargs: Any) -> InfrahubNode +get(self, kind: str, raise_when_missing: Literal[True], at: Timestamp | None = ..., branch: str | None = ..., timeout: int | None = ..., id: str | None = ..., hfid: list[str] | None = ..., include: list[str] | None = ..., exclude: list[str] | None = ..., populate_store: bool = ..., fragment: bool = ..., prefetch_relationships: bool = ..., property: bool = ..., include_metadata: bool = ..., query_name: str | None = ..., priority: Priority | None = ..., **kwargs: Any) -> InfrahubNode ``` #### `get` ```python -get(self, kind: str, raise_when_missing: bool = ..., at: Timestamp | None = ..., branch: str | None = ..., timeout: int | None = ..., id: str | None = ..., hfid: list[str] | None = ..., include: list[str] | None = ..., exclude: list[str] | None = ..., populate_store: bool = ..., fragment: bool = ..., prefetch_relationships: bool = ..., property: bool = ..., include_metadata: bool = ..., query_name: str | None = ..., **kwargs: Any) -> InfrahubNode +get(self, kind: str, raise_when_missing: bool = ..., at: Timestamp | None = ..., branch: str | None = ..., timeout: int | None = ..., id: str | None = ..., hfid: list[str] | None = ..., include: list[str] | None = ..., exclude: list[str] | None = ..., populate_store: bool = ..., fragment: bool = ..., prefetch_relationships: bool = ..., property: bool = ..., include_metadata: bool = ..., query_name: str | None = ..., priority: Priority | None = ..., **kwargs: Any) -> InfrahubNode ``` #### `get` ```python -get(self, kind: str | type[SchemaType], raise_when_missing: bool = True, at: Timestamp | None = None, branch: str | None = None, timeout: int | None = None, id: str | None = None, hfid: list[str] | None = None, include: list[str] | None = None, exclude: list[str] | None = None, populate_store: bool = True, fragment: bool = False, prefetch_relationships: bool = False, property: bool = False, include_metadata: bool = False, query_name: str | None = None, **kwargs: Any) -> InfrahubNode | SchemaType | None +get(self, kind: str | type[SchemaType], raise_when_missing: bool = True, at: Timestamp | None = None, branch: str | None = None, timeout: int | None = None, id: str | None = None, hfid: list[str] | None = None, include: list[str] | None = None, exclude: list[str] | None = None, populate_store: bool = True, fragment: bool = False, prefetch_relationships: bool = False, property: bool = False, include_metadata: bool = False, query_name: str | None = None, priority: Priority | None = None, **kwargs: Any) -> InfrahubNode | SchemaType | None ```
@@ -97,6 +97,14 @@ get_version(self) -> str Return the Infrahub version. +#### `get_server_information` + +```python +get_server_information(self) -> ServerInfo +``` + +Return the Infrahub server information (version and deployment ID). + #### `get_user` ```python @@ -116,7 +124,7 @@ Return user permissions. #### `count` ```python -count(self, kind: str | type[SchemaType], at: Timestamp | None = None, branch: str | None = None, timeout: int | None = None, partial_match: bool = False, query_name: str | None = None, **kwargs: Any) -> int +count(self, kind: str | type[SchemaType], at: Timestamp | None = None, branch: str | None = None, timeout: int | None = None, partial_match: bool = False, query_name: str | None = None, priority: Priority | None = None, **kwargs: Any) -> int ``` Return the number of nodes of a given kind. @@ -223,7 +231,7 @@ Requires Infrahub 1.10 or later. #### `all` ```python -all(self, kind: type[SchemaType], at: Timestamp | None = ..., branch: str | None = ..., timeout: int | None = ..., populate_store: bool = ..., offset: int | None = ..., limit: int | None = ..., include: list[str] | None = ..., exclude: list[str] | None = ..., fragment: bool = ..., prefetch_relationships: bool = ..., property: bool = ..., parallel: bool = ..., order: Order | None = ..., include_metadata: bool = ..., query_name: str | None = ...) -> list[SchemaType] +all(self, kind: type[SchemaType], at: Timestamp | None = ..., branch: str | None = ..., timeout: int | None = ..., populate_store: bool = ..., offset: int | None = ..., limit: int | None = ..., include: list[str] | None = ..., exclude: list[str] | None = ..., fragment: bool = ..., prefetch_relationships: bool = ..., property: bool = ..., parallel: bool = ..., order: Order | None = ..., include_metadata: bool = ..., query_name: str | None = ..., priority: Priority | None = ...) -> list[SchemaType] ```
@@ -232,13 +240,13 @@ all(self, kind: type[SchemaType], at: Timestamp | None = ..., branch: str | None #### `all` ```python -all(self, kind: str, at: Timestamp | None = ..., branch: str | None = ..., timeout: int | None = ..., populate_store: bool = ..., offset: int | None = ..., limit: int | None = ..., include: list[str] | None = ..., exclude: list[str] | None = ..., fragment: bool = ..., prefetch_relationships: bool = ..., property: bool = ..., parallel: bool = ..., order: Order | None = ..., include_metadata: bool = ..., query_name: str | None = ...) -> list[InfrahubNode] +all(self, kind: str, at: Timestamp | None = ..., branch: str | None = ..., timeout: int | None = ..., populate_store: bool = ..., offset: int | None = ..., limit: int | None = ..., include: list[str] | None = ..., exclude: list[str] | None = ..., fragment: bool = ..., prefetch_relationships: bool = ..., property: bool = ..., parallel: bool = ..., order: Order | None = ..., include_metadata: bool = ..., query_name: str | None = ..., priority: Priority | None = ...) -> list[InfrahubNode] ``` #### `all` ```python -all(self, kind: str | type[SchemaType], at: Timestamp | None = None, branch: str | None = None, timeout: int | None = None, populate_store: bool = True, offset: int | None = None, limit: int | None = None, include: list[str] | None = None, exclude: list[str] | None = None, fragment: bool = False, prefetch_relationships: bool = False, property: bool = False, parallel: bool = False, order: Order | None = None, include_metadata: bool = False, query_name: str | None = None) -> list[InfrahubNode] | list[SchemaType] +all(self, kind: str | type[SchemaType], at: Timestamp | None = None, branch: str | None = None, timeout: int | None = None, populate_store: bool = True, offset: int | None = None, limit: int | None = None, include: list[str] | None = None, exclude: list[str] | None = None, fragment: bool = False, prefetch_relationships: bool = False, property: bool = False, parallel: bool = False, order: Order | None = None, include_metadata: bool = False, query_name: str | None = None, priority: Priority | None = None) -> list[InfrahubNode] | list[SchemaType] ``` Retrieve all nodes of a given kind. @@ -260,6 +268,8 @@ Retrieve all nodes of a given kind. - `order`: Ordering related options. Setting `disable=True` enhances performances. - `include_metadata`: If True, includes node_metadata and relationship_metadata in the query. - `query_name`: If provided is used as the GraphQL operation name else All_<kind> is used. +- `priority`: Per-request priority emitted as the X-Priority header, overriding the +client default for these requests only. When None, the client default (if any) is used. **Returns:** @@ -270,7 +280,7 @@ Retrieve all nodes of a given kind. #### `filters` ```python -filters(self, kind: type[SchemaType], at: Timestamp | None = ..., branch: str | None = ..., timeout: int | None = ..., populate_store: bool = ..., offset: int | None = ..., limit: int | None = ..., include: list[str] | None = ..., exclude: list[str] | None = ..., fragment: bool = ..., prefetch_relationships: bool = ..., partial_match: bool = ..., property: bool = ..., parallel: bool = ..., order: Order | None = ..., include_metadata: bool = ..., query_name: str | None = ..., **kwargs: Any) -> list[SchemaType] +filters(self, kind: type[SchemaType], at: Timestamp | None = ..., branch: str | None = ..., timeout: int | None = ..., populate_store: bool = ..., offset: int | None = ..., limit: int | None = ..., include: list[str] | None = ..., exclude: list[str] | None = ..., fragment: bool = ..., prefetch_relationships: bool = ..., partial_match: bool = ..., property: bool = ..., parallel: bool = ..., order: Order | None = ..., include_metadata: bool = ..., query_name: str | None = ..., priority: Priority | None = ..., **kwargs: Any) -> list[SchemaType] ```
@@ -279,13 +289,13 @@ filters(self, kind: type[SchemaType], at: Timestamp | None = ..., branch: str | #### `filters` ```python -filters(self, kind: str, at: Timestamp | None = ..., branch: str | None = ..., timeout: int | None = ..., populate_store: bool = ..., offset: int | None = ..., limit: int | None = ..., include: list[str] | None = ..., exclude: list[str] | None = ..., fragment: bool = ..., prefetch_relationships: bool = ..., partial_match: bool = ..., property: bool = ..., parallel: bool = ..., order: Order | None = ..., include_metadata: bool = ..., query_name: str | None = ..., **kwargs: Any) -> list[InfrahubNode] +filters(self, kind: str, at: Timestamp | None = ..., branch: str | None = ..., timeout: int | None = ..., populate_store: bool = ..., offset: int | None = ..., limit: int | None = ..., include: list[str] | None = ..., exclude: list[str] | None = ..., fragment: bool = ..., prefetch_relationships: bool = ..., partial_match: bool = ..., property: bool = ..., parallel: bool = ..., order: Order | None = ..., include_metadata: bool = ..., query_name: str | None = ..., priority: Priority | None = ..., **kwargs: Any) -> list[InfrahubNode] ``` #### `filters` ```python -filters(self, kind: str | type[SchemaType], at: Timestamp | None = None, branch: str | None = None, timeout: int | None = None, populate_store: bool = True, offset: int | None = None, limit: int | None = None, include: list[str] | None = None, exclude: list[str] | None = None, fragment: bool = False, prefetch_relationships: bool = False, partial_match: bool = False, property: bool = False, parallel: bool = False, order: Order | None = None, include_metadata: bool = False, query_name: str | None = None, **kwargs: Any) -> list[InfrahubNode] | list[SchemaType] +filters(self, kind: str | type[SchemaType], at: Timestamp | None = None, branch: str | None = None, timeout: int | None = None, populate_store: bool = True, offset: int | None = None, limit: int | None = None, include: list[str] | None = None, exclude: list[str] | None = None, fragment: bool = False, prefetch_relationships: bool = False, partial_match: bool = False, property: bool = False, parallel: bool = False, order: Order | None = None, include_metadata: bool = False, query_name: str | None = None, priority: Priority | None = None, **kwargs: Any) -> list[InfrahubNode] | list[SchemaType] ``` Retrieve nodes of a given kind based on provided filters. @@ -308,6 +318,8 @@ Retrieve nodes of a given kind based on provided filters. - `order`: Ordering related options. Setting `disable=True` enhances performances. - `include_metadata`: If True, includes node_metadata and relationship_metadata in the query. - `query_name`: If provided is used as the GraphQL operation name else Filters_<kind> is used. +- `priority`: Per-request priority emitted as the X-Priority header, overriding the +client default for these requests only. When None, the client default (if any) is used. - `**kwargs`: Additional filter criteria for the query. **Returns:** @@ -327,7 +339,7 @@ Return a cloned version of the client using the same configuration. #### `execute_graphql` ```python -execute_graphql(self, query: str, variables: dict | None = None, branch_name: str | None = None, at: str | Timestamp | None = None, timeout: int | None = None, tracker: str | None = None, operation_name: str | None = None) -> dict +execute_graphql(self, query: str, variables: dict | None = None, branch_name: str | None = None, at: str | Timestamp | None = None, timeout: int | None = None, tracker: str | None = None, operation_name: str | None = None, priority: Priority | None = None) -> dict ``` Execute a GraphQL query (or mutation). @@ -343,6 +355,8 @@ If retry_on_failure is True, the query will retry until the server becomes reach - `timeout`: Timeout in second for the query. Defaults to None. - `operation_name`: GraphQL operation name, sent as `operationName` in the request payload so tracing/observability tools can identify the operation. Defaults to None. +- `priority`: Per-request priority emitted as the X-Priority header. Overrides the +client-wide default for this request only. When None, the client default (if any) is used. **Returns:** @@ -377,19 +391,19 @@ query_gql_query(self, name: str, variables: dict | None = None, update_group: bo #### `create_diff` ```python -create_diff(self, branch: str, name: str, from_time: datetime, to_time: datetime, wait_until_completion: bool = True) -> bool | str +create_diff(self, branch: str, name: str, from_time: datetime, to_time: datetime, wait_until_completion: bool = True, priority: Priority | None = None) -> bool | str ``` #### `get_diff_summary` ```python -get_diff_summary(self, branch: str, name: str | None = None, from_time: datetime | None = None, to_time: datetime | None = None, timeout: int | None = None, tracker: str | None = None) -> list[NodeDiff] +get_diff_summary(self, branch: str, name: str | None = None, from_time: datetime | None = None, to_time: datetime | None = None, timeout: int | None = None, tracker: str | None = None, priority: Priority | None = None) -> list[NodeDiff] ``` #### `get_diff_tree` ```python -get_diff_tree(self, branch: str, name: str | None = None, from_time: datetime | None = None, to_time: datetime | None = None, timeout: int | None = None, tracker: str | None = None) -> DiffTreeData | None +get_diff_tree(self, branch: str, name: str | None = None, from_time: datetime | None = None, to_time: datetime | None = None, timeout: int | None = None, tracker: str | None = None, priority: Priority | None = None) -> DiffTreeData | None ``` Get complete diff tree with metadata and nodes. @@ -526,7 +540,7 @@ mapping. See https://docs.infrahub.app/guides/object-conversion for more informa #### `get` ```python -get(self, kind: type[SchemaTypeSync], raise_when_missing: Literal[False], at: Timestamp | None = ..., branch: str | None = ..., timeout: int | None = ..., id: str | None = ..., hfid: list[str] | None = ..., include: list[str] | None = ..., exclude: list[str] | None = ..., populate_store: bool = ..., fragment: bool = ..., prefetch_relationships: bool = ..., property: bool = ..., include_metadata: bool = ..., query_name: str | None = ..., **kwargs: Any) -> SchemaTypeSync | None +get(self, kind: type[SchemaTypeSync], raise_when_missing: Literal[False], at: Timestamp | None = ..., branch: str | None = ..., timeout: int | None = ..., id: str | None = ..., hfid: list[str] | None = ..., include: list[str] | None = ..., exclude: list[str] | None = ..., populate_store: bool = ..., fragment: bool = ..., prefetch_relationships: bool = ..., property: bool = ..., include_metadata: bool = ..., query_name: str | None = ..., priority: Priority | None = ..., **kwargs: Any) -> SchemaTypeSync | None ```
@@ -535,37 +549,37 @@ get(self, kind: type[SchemaTypeSync], raise_when_missing: Literal[False], at: Ti #### `get` ```python -get(self, kind: type[SchemaTypeSync], raise_when_missing: Literal[True], at: Timestamp | None = ..., branch: str | None = ..., timeout: int | None = ..., id: str | None = ..., hfid: list[str] | None = ..., include: list[str] | None = ..., exclude: list[str] | None = ..., populate_store: bool = ..., fragment: bool = ..., prefetch_relationships: bool = ..., property: bool = ..., include_metadata: bool = ..., query_name: str | None = ..., **kwargs: Any) -> SchemaTypeSync +get(self, kind: type[SchemaTypeSync], raise_when_missing: Literal[True], at: Timestamp | None = ..., branch: str | None = ..., timeout: int | None = ..., id: str | None = ..., hfid: list[str] | None = ..., include: list[str] | None = ..., exclude: list[str] | None = ..., populate_store: bool = ..., fragment: bool = ..., prefetch_relationships: bool = ..., property: bool = ..., include_metadata: bool = ..., query_name: str | None = ..., priority: Priority | None = ..., **kwargs: Any) -> SchemaTypeSync ``` #### `get` ```python -get(self, kind: type[SchemaTypeSync], raise_when_missing: bool = ..., at: Timestamp | None = ..., branch: str | None = ..., timeout: int | None = ..., id: str | None = ..., hfid: list[str] | None = ..., include: list[str] | None = ..., exclude: list[str] | None = ..., populate_store: bool = ..., fragment: bool = ..., prefetch_relationships: bool = ..., property: bool = ..., include_metadata: bool = ..., query_name: str | None = ..., **kwargs: Any) -> SchemaTypeSync +get(self, kind: type[SchemaTypeSync], raise_when_missing: bool = ..., at: Timestamp | None = ..., branch: str | None = ..., timeout: int | None = ..., id: str | None = ..., hfid: list[str] | None = ..., include: list[str] | None = ..., exclude: list[str] | None = ..., populate_store: bool = ..., fragment: bool = ..., prefetch_relationships: bool = ..., property: bool = ..., include_metadata: bool = ..., query_name: str | None = ..., priority: Priority | None = ..., **kwargs: Any) -> SchemaTypeSync ``` #### `get` ```python -get(self, kind: str, raise_when_missing: Literal[False], at: Timestamp | None = ..., branch: str | None = ..., timeout: int | None = ..., id: str | None = ..., hfid: list[str] | None = ..., include: list[str] | None = ..., exclude: list[str] | None = ..., populate_store: bool = ..., fragment: bool = ..., prefetch_relationships: bool = ..., property: bool = ..., include_metadata: bool = ..., query_name: str | None = ..., **kwargs: Any) -> InfrahubNodeSync | None +get(self, kind: str, raise_when_missing: Literal[False], at: Timestamp | None = ..., branch: str | None = ..., timeout: int | None = ..., id: str | None = ..., hfid: list[str] | None = ..., include: list[str] | None = ..., exclude: list[str] | None = ..., populate_store: bool = ..., fragment: bool = ..., prefetch_relationships: bool = ..., property: bool = ..., include_metadata: bool = ..., query_name: str | None = ..., priority: Priority | None = ..., **kwargs: Any) -> InfrahubNodeSync | None ``` #### `get` ```python -get(self, kind: str, raise_when_missing: Literal[True], at: Timestamp | None = ..., branch: str | None = ..., timeout: int | None = ..., id: str | None = ..., hfid: list[str] | None = ..., include: list[str] | None = ..., exclude: list[str] | None = ..., populate_store: bool = ..., fragment: bool = ..., prefetch_relationships: bool = ..., property: bool = ..., include_metadata: bool = ..., query_name: str | None = ..., **kwargs: Any) -> InfrahubNodeSync +get(self, kind: str, raise_when_missing: Literal[True], at: Timestamp | None = ..., branch: str | None = ..., timeout: int | None = ..., id: str | None = ..., hfid: list[str] | None = ..., include: list[str] | None = ..., exclude: list[str] | None = ..., populate_store: bool = ..., fragment: bool = ..., prefetch_relationships: bool = ..., property: bool = ..., include_metadata: bool = ..., query_name: str | None = ..., priority: Priority | None = ..., **kwargs: Any) -> InfrahubNodeSync ``` #### `get` ```python -get(self, kind: str, raise_when_missing: bool = ..., at: Timestamp | None = ..., branch: str | None = ..., timeout: int | None = ..., id: str | None = ..., hfid: list[str] | None = ..., include: list[str] | None = ..., exclude: list[str] | None = ..., populate_store: bool = ..., fragment: bool = ..., prefetch_relationships: bool = ..., property: bool = ..., include_metadata: bool = ..., query_name: str | None = ..., **kwargs: Any) -> InfrahubNodeSync +get(self, kind: str, raise_when_missing: bool = ..., at: Timestamp | None = ..., branch: str | None = ..., timeout: int | None = ..., id: str | None = ..., hfid: list[str] | None = ..., include: list[str] | None = ..., exclude: list[str] | None = ..., populate_store: bool = ..., fragment: bool = ..., prefetch_relationships: bool = ..., property: bool = ..., include_metadata: bool = ..., query_name: str | None = ..., priority: Priority | None = ..., **kwargs: Any) -> InfrahubNodeSync ``` #### `get` ```python -get(self, kind: str | type[SchemaTypeSync], raise_when_missing: bool = True, at: Timestamp | None = None, branch: str | None = None, timeout: int | None = None, id: str | None = None, hfid: list[str] | None = None, include: list[str] | None = None, exclude: list[str] | None = None, populate_store: bool = True, fragment: bool = False, prefetch_relationships: bool = False, property: bool = False, include_metadata: bool = False, query_name: str | None = None, **kwargs: Any) -> InfrahubNodeSync | SchemaTypeSync | None +get(self, kind: str | type[SchemaTypeSync], raise_when_missing: bool = True, at: Timestamp | None = None, branch: str | None = None, timeout: int | None = None, id: str | None = None, hfid: list[str] | None = None, include: list[str] | None = None, exclude: list[str] | None = None, populate_store: bool = True, fragment: bool = False, prefetch_relationships: bool = False, property: bool = False, include_metadata: bool = False, query_name: str | None = None, priority: Priority | None = None, **kwargs: Any) -> InfrahubNodeSync | SchemaTypeSync | None ```
@@ -607,6 +621,14 @@ get_version(self) -> str Return the Infrahub version. +#### `get_server_information` + +```python +get_server_information(self) -> ServerInfo +``` + +Return the Infrahub server information (version and deployment ID). + #### `get_user` ```python @@ -634,7 +656,7 @@ Return a cloned version of the client using the same configuration. #### `execute_graphql` ```python -execute_graphql(self, query: str, variables: dict | None = None, branch_name: str | None = None, at: str | Timestamp | None = None, timeout: int | None = None, tracker: str | None = None, operation_name: str | None = None) -> dict +execute_graphql(self, query: str, variables: dict | None = None, branch_name: str | None = None, at: str | Timestamp | None = None, timeout: int | None = None, tracker: str | None = None, operation_name: str | None = None, priority: Priority | None = None) -> dict ``` Execute a GraphQL query (or mutation). @@ -650,6 +672,8 @@ If retry_on_failure is True, the query will retry until the server becomes reach - `timeout`: Timeout in second for the query. Defaults to None. - `operation_name`: GraphQL operation name, sent as `operationName` in the request payload so tracing/observability tools can identify the operation. Defaults to None. +- `priority`: Per-request priority emitted as the X-Priority header. Overrides the +client-wide default for this request only. When None, the client default (if any) is used. **Returns:** @@ -666,7 +690,7 @@ so tracing/observability tools can identify the operation. Defaults to None. #### `count` ```python -count(self, kind: str | type[SchemaType], at: Timestamp | None = None, branch: str | None = None, timeout: int | None = None, partial_match: bool = False, query_name: str | None = None, **kwargs: Any) -> int +count(self, kind: str | type[SchemaType], at: Timestamp | None = None, branch: str | None = None, timeout: int | None = None, partial_match: bool = False, query_name: str | None = None, priority: Priority | None = None, **kwargs: Any) -> int ``` Return the number of nodes of a given kind. @@ -773,7 +797,7 @@ Requires Infrahub 1.10 or later. #### `all` ```python -all(self, kind: type[SchemaTypeSync], at: Timestamp | None = ..., branch: str | None = ..., timeout: int | None = ..., populate_store: bool = ..., offset: int | None = ..., limit: int | None = ..., include: list[str] | None = ..., exclude: list[str] | None = ..., fragment: bool = ..., prefetch_relationships: bool = ..., property: bool = ..., parallel: bool = ..., order: Order | None = ..., include_metadata: bool = ..., query_name: str | None = ...) -> list[SchemaTypeSync] +all(self, kind: type[SchemaTypeSync], at: Timestamp | None = ..., branch: str | None = ..., timeout: int | None = ..., populate_store: bool = ..., offset: int | None = ..., limit: int | None = ..., include: list[str] | None = ..., exclude: list[str] | None = ..., fragment: bool = ..., prefetch_relationships: bool = ..., property: bool = ..., parallel: bool = ..., order: Order | None = ..., include_metadata: bool = ..., query_name: str | None = ..., priority: Priority | None = ...) -> list[SchemaTypeSync] ```
@@ -782,13 +806,13 @@ all(self, kind: type[SchemaTypeSync], at: Timestamp | None = ..., branch: str | #### `all` ```python -all(self, kind: str, at: Timestamp | None = ..., branch: str | None = ..., timeout: int | None = ..., populate_store: bool = ..., offset: int | None = ..., limit: int | None = ..., include: list[str] | None = ..., exclude: list[str] | None = ..., fragment: bool = ..., prefetch_relationships: bool = ..., property: bool = ..., parallel: bool = ..., order: Order | None = ..., include_metadata: bool = ..., query_name: str | None = ...) -> list[InfrahubNodeSync] +all(self, kind: str, at: Timestamp | None = ..., branch: str | None = ..., timeout: int | None = ..., populate_store: bool = ..., offset: int | None = ..., limit: int | None = ..., include: list[str] | None = ..., exclude: list[str] | None = ..., fragment: bool = ..., prefetch_relationships: bool = ..., property: bool = ..., parallel: bool = ..., order: Order | None = ..., include_metadata: bool = ..., query_name: str | None = ..., priority: Priority | None = ...) -> list[InfrahubNodeSync] ``` #### `all` ```python -all(self, kind: str | type[SchemaTypeSync], at: Timestamp | None = None, branch: str | None = None, timeout: int | None = None, populate_store: bool = True, offset: int | None = None, limit: int | None = None, include: list[str] | None = None, exclude: list[str] | None = None, fragment: bool = False, prefetch_relationships: bool = False, property: bool = False, parallel: bool = False, order: Order | None = None, include_metadata: bool = False, query_name: str | None = None) -> list[InfrahubNodeSync] | list[SchemaTypeSync] +all(self, kind: str | type[SchemaTypeSync], at: Timestamp | None = None, branch: str | None = None, timeout: int | None = None, populate_store: bool = True, offset: int | None = None, limit: int | None = None, include: list[str] | None = None, exclude: list[str] | None = None, fragment: bool = False, prefetch_relationships: bool = False, property: bool = False, parallel: bool = False, order: Order | None = None, include_metadata: bool = False, query_name: str | None = None, priority: Priority | None = None) -> list[InfrahubNodeSync] | list[SchemaTypeSync] ``` Retrieve all nodes of a given kind. @@ -810,6 +834,8 @@ Retrieve all nodes of a given kind. - `order`: Ordering related options. Setting `disable=True` enhances performances. - `include_metadata`: If True, includes node_metadata and relationship_metadata in the query. - `query_name`: If provided is used as the GraphQL operation name else All_<kind> is used. +- `priority`: Per-request priority emitted as the X-Priority header, overriding the +client default for these requests only. When None, the client default (if any) is used. **Returns:** @@ -820,7 +846,7 @@ Retrieve all nodes of a given kind. #### `filters` ```python -filters(self, kind: type[SchemaTypeSync], at: Timestamp | None = ..., branch: str | None = ..., timeout: int | None = ..., populate_store: bool = ..., offset: int | None = ..., limit: int | None = ..., include: list[str] | None = ..., exclude: list[str] | None = ..., fragment: bool = ..., prefetch_relationships: bool = ..., partial_match: bool = ..., property: bool = ..., parallel: bool = ..., order: Order | None = ..., include_metadata: bool = ..., query_name: str | None = ..., **kwargs: Any) -> list[SchemaTypeSync] +filters(self, kind: type[SchemaTypeSync], at: Timestamp | None = ..., branch: str | None = ..., timeout: int | None = ..., populate_store: bool = ..., offset: int | None = ..., limit: int | None = ..., include: list[str] | None = ..., exclude: list[str] | None = ..., fragment: bool = ..., prefetch_relationships: bool = ..., partial_match: bool = ..., property: bool = ..., parallel: bool = ..., order: Order | None = ..., include_metadata: bool = ..., query_name: str | None = ..., priority: Priority | None = ..., **kwargs: Any) -> list[SchemaTypeSync] ```
@@ -829,13 +855,13 @@ filters(self, kind: type[SchemaTypeSync], at: Timestamp | None = ..., branch: st #### `filters` ```python -filters(self, kind: str, at: Timestamp | None = ..., branch: str | None = ..., timeout: int | None = ..., populate_store: bool = ..., offset: int | None = ..., limit: int | None = ..., include: list[str] | None = ..., exclude: list[str] | None = ..., fragment: bool = ..., prefetch_relationships: bool = ..., partial_match: bool = ..., property: bool = ..., parallel: bool = ..., order: Order | None = ..., include_metadata: bool = ..., query_name: str | None = ..., **kwargs: Any) -> list[InfrahubNodeSync] +filters(self, kind: str, at: Timestamp | None = ..., branch: str | None = ..., timeout: int | None = ..., populate_store: bool = ..., offset: int | None = ..., limit: int | None = ..., include: list[str] | None = ..., exclude: list[str] | None = ..., fragment: bool = ..., prefetch_relationships: bool = ..., partial_match: bool = ..., property: bool = ..., parallel: bool = ..., order: Order | None = ..., include_metadata: bool = ..., query_name: str | None = ..., priority: Priority | None = ..., **kwargs: Any) -> list[InfrahubNodeSync] ``` #### `filters` ```python -filters(self, kind: str | type[SchemaTypeSync], at: Timestamp | None = None, branch: str | None = None, timeout: int | None = None, populate_store: bool = True, offset: int | None = None, limit: int | None = None, include: list[str] | None = None, exclude: list[str] | None = None, fragment: bool = False, prefetch_relationships: bool = False, partial_match: bool = False, property: bool = False, parallel: bool = False, order: Order | None = None, include_metadata: bool = False, query_name: str | None = None, **kwargs: Any) -> list[InfrahubNodeSync] | list[SchemaTypeSync] +filters(self, kind: str | type[SchemaTypeSync], at: Timestamp | None = None, branch: str | None = None, timeout: int | None = None, populate_store: bool = True, offset: int | None = None, limit: int | None = None, include: list[str] | None = None, exclude: list[str] | None = None, fragment: bool = False, prefetch_relationships: bool = False, partial_match: bool = False, property: bool = False, parallel: bool = False, order: Order | None = None, include_metadata: bool = False, query_name: str | None = None, priority: Priority | None = None, **kwargs: Any) -> list[InfrahubNodeSync] | list[SchemaTypeSync] ``` Retrieve nodes of a given kind based on provided filters. @@ -858,6 +884,8 @@ Retrieve nodes of a given kind based on provided filters. - `order`: Ordering related options. Setting `disable=True` enhances performances. - `include_metadata`: If True, includes node_metadata and relationship_metadata in the query. - `query_name`: If provided is used as the GraphQL operation name else Filters_<kind> is used. +- `priority`: Per-request priority emitted as the X-Priority header, overriding the +client default for these requests only. When None, the client default (if any) is used. - `**kwargs`: Additional filter criteria for the query. **Returns:** @@ -892,19 +920,19 @@ query_gql_query(self, name: str, variables: dict | None = None, update_group: bo #### `create_diff` ```python -create_diff(self, branch: str, name: str, from_time: datetime, to_time: datetime, wait_until_completion: bool = True) -> bool | str +create_diff(self, branch: str, name: str, from_time: datetime, to_time: datetime, wait_until_completion: bool = True, priority: Priority | None = None) -> bool | str ``` #### `get_diff_summary` ```python -get_diff_summary(self, branch: str, name: str | None = None, from_time: datetime | None = None, to_time: datetime | None = None, timeout: int | None = None, tracker: str | None = None) -> list[NodeDiff] +get_diff_summary(self, branch: str, name: str | None = None, from_time: datetime | None = None, to_time: datetime | None = None, timeout: int | None = None, tracker: str | None = None, priority: Priority | None = None) -> list[NodeDiff] ``` #### `get_diff_tree` ```python -get_diff_tree(self, branch: str, name: str | None = None, from_time: datetime | None = None, to_time: datetime | None = None, timeout: int | None = None, tracker: str | None = None) -> DiffTreeData | None +get_diff_tree(self, branch: str, name: str | None = None, from_time: datetime | None = None, to_time: datetime | None = None, timeout: int | None = None, tracker: str | None = None, priority: Priority | None = None) -> DiffTreeData | None ``` Get complete diff tree with metadata and nodes. @@ -986,7 +1014,7 @@ Allocate a new IP prefix by using the provided resource pool. - `resource_pool`: Node corresponding to the pool to allocate resources from. - `identifier`: Value to perform idempotent allocation, the same resource will be returned for a given identifier. -- `size`: Length of the prefix to allocate. +- `prefix_length`: Length of the prefix to allocate. - `member_type`: Member type of the prefix to allocate. - `prefix_type`: Kind of the prefix to allocate. - `data`: A key/value map to use to set attributes values on the allocated prefix. diff --git a/docs/docs/python-sdk/sdk_ref/infrahub_sdk/node/node.mdx b/docs/docs/python-sdk/sdk_ref/infrahub_sdk/node/node.mdx index 99671d1ed..ef2823562 100644 --- a/docs/docs/python-sdk/sdk_ref/infrahub_sdk/node/node.mdx +++ b/docs/docs/python-sdk/sdk_ref/infrahub_sdk/node/node.mdx @@ -257,7 +257,7 @@ from ``source.name`` when ``name`` is omitted. #### `delete` ```python -delete(self, timeout: int | None = None, request_context: RequestContext | None = None) -> None +delete(self, timeout: int | None = None, request_context: RequestContext | None = None, priority: Priority | None = None) -> None ``` Delete this node on the backend. @@ -268,11 +268,13 @@ Delete this node on the backend. GraphQL API. Specified in seconds. - `request_context`: Request-level context passed through to the mutation. When omitted, the client's request context is used. +- `priority`: Per-request priority emitted as the X-Priority header, +overriding the client default for this request only. #### `save` ```python -save(self, allow_upsert: bool = False, update_group_context: bool | None = None, timeout: int | None = None, request_context: RequestContext | None = None) -> None +save(self, allow_upsert: bool = False, update_group_context: bool | None = None, timeout: int | None = None, request_context: RequestContext | None = None, priority: Priority | None = None) -> None ``` Persist this node to the backend, creating or updating it as appropriate. @@ -293,11 +295,13 @@ to ``True``. GraphQL API. Specified in seconds. - `request_context`: Request-level context passed through to the mutation. When omitted, the client's request context is used. +- `priority`: Per-request priority emitted as the X-Priority header, +overriding the client default for this request only. #### `generate_query_data` ```python -generate_query_data(self, filters: dict[str, Any] | None = None, offset: int | None = None, limit: int | None = None, include: list[str] | None = None, exclude: list[str] | None = None, fragment: bool = False, prefetch_relationships: bool = False, partial_match: bool = False, property: bool = False, order: Order | None = None, include_metadata: bool = False) -> dict[str, Any | dict] +generate_query_data(self, filters: dict[str, Any] | None = None, offset: int | str | None = None, limit: int | str | None = None, include: list[str] | None = None, exclude: list[str] | None = None, fragment: bool = False, prefetch_relationships: bool = False, partial_match: bool = False, property: bool = False, order: Order | None = None, include_metadata: bool = False) -> dict[str, Any | dict] ``` Generate the full GraphQL query payload for this node kind. @@ -310,8 +314,10 @@ relevant attributes are returned alongside the generic fields. **Args:** - `filters`: Filters to apply to the query. -- `offset`: Pagination offset. -- `limit`: Pagination limit. +- `offset`: Pagination offset, either a literal value or a +GraphQL variable placeholder such as ``"$offset"``. +- `limit`: Pagination limit, either a literal value or a +GraphQL variable placeholder such as ``"$limit"``. - `include`: Attributes or relationships to include. - `exclude`: Attributes or relationships to exclude. - `fragment`: When ``True`` and the schema is a generic, emit @@ -388,7 +394,7 @@ the listed peers, leaving every other field untouched. #### `create` ```python -create(self, allow_upsert: bool = False, timeout: int | None = None, request_context: RequestContext | None = None) -> None +create(self, allow_upsert: bool = False, timeout: int | None = None, request_context: RequestContext | None = None, priority: Priority | None = None) -> None ``` Create this node on the backend. @@ -408,6 +414,8 @@ erroring on a duplicate. Defaults to ``False``. GraphQL API. Specified in seconds. - `request_context`: Request-level context passed through to the mutation. When omitted, the client's request context is used. +- `priority`: Per-request priority emitted as the X-Priority header, +overriding the client default for this request only. **Raises:** @@ -416,7 +424,7 @@ to the mutation. When omitted, the client's request context is used. #### `update` ```python -update(self, do_full_update: bool = False, timeout: int | None = None, request_context: RequestContext | None = None) -> None +update(self, do_full_update: bool = False, timeout: int | None = None, request_context: RequestContext | None = None, priority: Priority | None = None) -> None ``` Update this node on the backend. @@ -436,6 +444,8 @@ unmodified. Defaults to ``False``. GraphQL API. Specified in seconds. - `request_context`: Request-level context passed through to the mutation. When omitted, the client's request context is used. +- `priority`: Per-request priority emitted as the X-Priority header, +overriding the client default for this request only. #### `get_pool_allocated_resources` @@ -776,7 +786,7 @@ from ``source.name`` when ``name`` is omitted. #### `delete` ```python -delete(self, timeout: int | None = None, request_context: RequestContext | None = None) -> None +delete(self, timeout: int | None = None, request_context: RequestContext | None = None, priority: Priority | None = None) -> None ``` Delete this node on the backend. @@ -787,11 +797,13 @@ Delete this node on the backend. GraphQL API. Specified in seconds. - `request_context`: Request-level context passed through to the mutation. When omitted, the client's request context is used. +- `priority`: Per-request priority emitted as the X-Priority header, +overriding the client default for this request only. #### `save` ```python -save(self, allow_upsert: bool = False, update_group_context: bool | None = None, timeout: int | None = None, request_context: RequestContext | None = None) -> None +save(self, allow_upsert: bool = False, update_group_context: bool | None = None, timeout: int | None = None, request_context: RequestContext | None = None, priority: Priority | None = None) -> None ``` Persist this node to the backend, creating or updating it as appropriate. @@ -812,11 +824,13 @@ to ``True``. GraphQL API. Specified in seconds. - `request_context`: Request-level context passed through to the mutation. When omitted, the client's request context is used. +- `priority`: Per-request priority emitted as the X-Priority header, +overriding the client default for this request only. #### `generate_query_data` ```python -generate_query_data(self, filters: dict[str, Any] | None = None, offset: int | None = None, limit: int | None = None, include: list[str] | None = None, exclude: list[str] | None = None, fragment: bool = False, prefetch_relationships: bool = False, partial_match: bool = False, property: bool = False, order: Order | None = None, include_metadata: bool = False) -> dict[str, Any | dict] +generate_query_data(self, filters: dict[str, Any] | None = None, offset: int | str | None = None, limit: int | str | None = None, include: list[str] | None = None, exclude: list[str] | None = None, fragment: bool = False, prefetch_relationships: bool = False, partial_match: bool = False, property: bool = False, order: Order | None = None, include_metadata: bool = False) -> dict[str, Any | dict] ``` Generate the full GraphQL query payload for this node kind. @@ -829,8 +843,10 @@ relevant attributes are returned alongside the generic fields. **Args:** - `filters`: Filters to apply to the query. -- `offset`: Pagination offset. -- `limit`: Pagination limit. +- `offset`: Pagination offset, either a literal value or a +GraphQL variable placeholder such as ``"$offset"``. +- `limit`: Pagination limit, either a literal value or a +GraphQL variable placeholder such as ``"$limit"``. - `include`: Attributes or relationships to include. - `exclude`: Attributes or relationships to exclude. - `fragment`: When ``True`` and the schema is a generic, emit @@ -907,7 +923,7 @@ the listed peers, leaving every other field untouched. #### `create` ```python -create(self, allow_upsert: bool = False, timeout: int | None = None, request_context: RequestContext | None = None) -> None +create(self, allow_upsert: bool = False, timeout: int | None = None, request_context: RequestContext | None = None, priority: Priority | None = None) -> None ``` Create this node on the backend. @@ -927,6 +943,8 @@ erroring on a duplicate. Defaults to ``False``. GraphQL API. Specified in seconds. - `request_context`: Request-level context passed through to the mutation. When omitted, the client's request context is used. +- `priority`: Per-request priority emitted as the X-Priority header, +overriding the client default for this request only. **Raises:** @@ -935,7 +953,7 @@ to the mutation. When omitted, the client's request context is used. #### `update` ```python -update(self, do_full_update: bool = False, timeout: int | None = None, request_context: RequestContext | None = None) -> None +update(self, do_full_update: bool = False, timeout: int | None = None, request_context: RequestContext | None = None, priority: Priority | None = None) -> None ``` Update this node on the backend. @@ -955,6 +973,8 @@ unmodified. Defaults to ``False``. GraphQL API. Specified in seconds. - `request_context`: Request-level context passed through to the mutation. When omitted, the client's request context is used. +- `priority`: Per-request priority emitted as the X-Priority header, +overriding the client default for this request only. #### `get_pool_allocated_resources` @@ -1336,7 +1356,7 @@ Return the raw GraphQL payload used to build this node. #### `generate_query_data_init` ```python -generate_query_data_init(self, filters: dict[str, Any] | None = None, offset: int | None = None, limit: int | None = None, include: list[str] | None = None, exclude: list[str] | None = None, partial_match: bool = False, order: Order | None = None, include_metadata: bool = False) -> dict[str, Any | dict] +generate_query_data_init(self, filters: dict[str, Any] | None = None, offset: int | str | None = None, limit: int | str | None = None, include: list[str] | None = None, exclude: list[str] | None = None, partial_match: bool = False, order: Order | None = None, include_metadata: bool = False) -> dict[str, Any | dict] ``` Build the top-level ``count``/``edges`` skeleton of a GraphQL query for this kind. @@ -1348,8 +1368,10 @@ The returned dict is the outer structure consumed by **Args:** - `filters`: Filters to apply to the query. -- `offset`: Pagination offset. -- `limit`: Pagination limit. +- `offset`: Pagination offset, either a literal value or a +GraphQL variable placeholder such as ``"$offset"``. +- `limit`: Pagination limit, either a literal value or a +GraphQL variable placeholder such as ``"$limit"``. - `include`: Attributes or relationships to include. - `exclude`: Attributes or relationships to exclude. - `partial_match`: When ``True``, allow partial matches on filter diff --git a/docs/docs/python-sdk/sdk_ref/infrahub_sdk/node/related_node.mdx b/docs/docs/python-sdk/sdk_ref/infrahub_sdk/node/related_node.mdx index 80e6fdd92..a79c1ad9a 100644 --- a/docs/docs/python-sdk/sdk_ref/infrahub_sdk/node/related_node.mdx +++ b/docs/docs/python-sdk/sdk_ref/infrahub_sdk/node/related_node.mdx @@ -173,7 +173,7 @@ around :meth:`get`. #### `fetch` ```python -fetch(self, timeout: int | None = None) -> None +fetch(self, timeout: int | None = None, priority: Priority | None = None) -> None ``` Fetch the full peer node from the backend and cache it on this object. @@ -185,6 +185,7 @@ available via :attr:`peer` or :meth:`get`. - `timeout`: Overrides the default timeout used when querying the GraphQL API. Specified in seconds. +- `priority`: Override the client-wide request priority for this fetch. When None, the client default is used. **Raises:** @@ -245,7 +246,7 @@ store, and :attr:`peer` is a convenience accessor around :meth:`get`. #### `fetch` ```python -fetch(self, timeout: int | None = None) -> None +fetch(self, timeout: int | None = None, priority: Priority | None = None) -> None ``` Fetch the full peer node from the backend and cache it on this object. @@ -257,6 +258,7 @@ available via :attr:`peer` or :meth:`get`. - `timeout`: Overrides the default timeout used when querying the GraphQL API. Specified in seconds. +- `priority`: Override the client-wide request priority for this fetch. When None, the client default is used. **Raises:** diff --git a/infrahub_sdk/branch.py b/infrahub_sdk/branch.py index d62ef23df..b310522e4 100644 --- a/infrahub_sdk/branch.py +++ b/infrahub_sdk/branch.py @@ -1,14 +1,12 @@ from __future__ import annotations from enum import Enum -from typing import TYPE_CHECKING, Any, Literal, overload -from urllib.parse import urlencode +from typing import TYPE_CHECKING, Literal, overload from pydantic import BaseModel from .exceptions import BranchNotFoundError from .graphql import Mutation, Query -from .utils import decode_json if TYPE_CHECKING: from .client import InfrahubClient, InfrahubClientSync @@ -20,6 +18,7 @@ class BranchStatus(str, Enum): NEED_UPGRADE_REBASE = "NEED_UPGRADE_REBASE" DELETING = "DELETING" MERGING = "MERGING" + MERGE_FAILED = "MERGE_FAILED" MERGED = "MERGED" @@ -60,30 +59,7 @@ class BranchData(BaseModel): QUERY_ONE_BRANCH_DATA = {"Branch": {**BRANCH_DATA, **BRANCH_DATA_FILTER}} -class InfraHubBranchManagerBase: - @classmethod - def generate_diff_data_url( - cls, - client: InfrahubClient | InfrahubClientSync, - branch_name: str, - branch_only: bool = True, - time_from: str | None = None, - time_to: str | None = None, - ) -> str: - """Generate the URL for the diff_data function.""" - url = f"{client.address}/api/diff/data" - url_params = {} - url_params["branch"] = branch_name - url_params["branch_only"] = str(branch_only).lower() - if time_from: - url_params["time_from"] = time_from - if time_to: - url_params["time_to"] = time_to - - return url + urlencode(url_params) - - -class InfrahubBranchManager(InfraHubBranchManagerBase): +class InfrahubBranchManager: def __init__(self, client: InfrahubClient) -> None: self.client = client @@ -205,25 +181,8 @@ async def get(self, branch_name: str) -> BranchData: raise BranchNotFoundError(identifier=branch_name) return BranchData(**data["Branch"][0]) - async def diff_data( - self, - branch_name: str, - branch_only: bool = True, - time_from: str | None = None, - time_to: str | None = None, - ) -> dict[Any, Any]: - url = self.generate_diff_data_url( - client=self.client, - branch_name=branch_name, - branch_only=branch_only, - time_from=time_from, - time_to=time_to, - ) - response = await self.client._get(url=url, headers=self.client.headers) - return decode_json(response=response) - -class InfrahubBranchManagerSync(InfraHubBranchManagerBase): +class InfrahubBranchManagerSync: def __init__(self, client: InfrahubClientSync) -> None: self.client = client @@ -299,23 +258,6 @@ def delete(self, branch_name: str) -> bool: response = self.client.execute_graphql(query=query.render(), tracker="mutation-branch-delete") return response["BranchDelete"]["ok"] - def diff_data( - self, - branch_name: str, - branch_only: bool = True, - time_from: str | None = None, - time_to: str | None = None, - ) -> dict[Any, Any]: - url = self.generate_diff_data_url( - client=self.client, - branch_name=branch_name, - branch_only=branch_only, - time_from=time_from, - time_to=time_to, - ) - response = self.client._get(url=url, headers=self.client.headers) - return decode_json(response=response) - def merge(self, branch_name: str) -> bool: input_data = { "data": { diff --git a/infrahub_sdk/client.py b/infrahub_sdk/client.py index f39458531..d600ea2fc 100644 --- a/infrahub_sdk/client.py +++ b/infrahub_sdk/client.py @@ -5,7 +5,7 @@ import logging import time from collections.abc import AsyncIterator, Callable, Coroutine, Iterable, Iterator, Mapping, MutableMapping -from contextlib import asynccontextmanager, contextmanager +from contextlib import AsyncExitStack, ExitStack, asynccontextmanager, contextmanager, suppress from datetime import datetime from enum import Enum from functools import wraps @@ -20,9 +20,9 @@ from .batch import InfrahubBatch, InfrahubBatchSync from .branch import MUTATION_QUERY_TASK, BranchData, InfrahubBranchManager, InfrahubBranchManagerSync from .config import Config -from .constants import InfrahubClientMode +from .constants import InfrahubClientMode, Priority from .convert_object_type import CONVERT_OBJECT_MUTATION, ConversionFieldInput -from .data import RepositoryBranchInfo, RepositoryData +from .data import RepositoryBranchInfo, RepositoryData, ServerInfo from .diff import DiffTreeData, NodeDiff, diff_tree_node_to_node_diff, get_diff_summary_query, get_diff_tree_query from .exceptions import ( AuthenticationError, @@ -49,6 +49,7 @@ from .protocols_base import CoreNode, CoreNodeSync from .queries import QUERY_USER, get_commit_update_mutation from .query_groups import InfrahubGroupContext, InfrahubGroupContextSync +from .rate_limit import RateLimitRetryHandler from .schema import InfrahubSchema, InfrahubSchemaSync, NodeSchemaAPI from .store import NodeStore, NodeStoreSync from .task.manager import InfrahubTaskManager, InfrahubTaskManagerSync @@ -79,6 +80,20 @@ class ProxyConfig(TypedDict): mounts: Mapping[str, AsyncBaseTransport | None] | None +def _rewind_multipart_files(files: dict[str, Any]) -> None: + """Rewind seekable file objects in a multipart ``files`` payload to position 0. + + httpx reads file-like objects to EOF on send; rewinding before each attempt lets a retried + upload (e.g. after a 429) carry the full body rather than an already-consumed stream. + """ + for value in files.values(): + file_obj = value[1] if isinstance(value, tuple) and len(value) > 1 else value + seek = getattr(file_obj, "seek", None) + if callable(seek): + with suppress(OSError, ValueError): # non-seekable stream: leave as-is + seek(0) + + class ProxyConfigSync(TypedDict): proxy: ProxyTypes | None mounts: Mapping[str, BaseTransport | None] | None @@ -186,6 +201,13 @@ def __init__( self.config.address = address or self.config.address self.insert_tracker = self.config.insert_tracker self.log = self.config.logger or logging.getLogger("infrahub_sdk") + self._rate_limit_handler = RateLimitRetryHandler( + max_retries=self.config.rate_limit_max_retries, + backoff_base=self.config.rate_limit_backoff_base, + backoff_max=self.config.rate_limit_backoff_max, + enabled=self.config.rate_limit_retry_enabled, + log=self.log, + ) self.address = self.config.address self.mode = self.config.mode self.pagination_size = self.config.pagination_size @@ -195,6 +217,9 @@ def __init__( if self.config.api_token: self.headers["X-INFRAHUB-KEY"] = self.config.api_token + if self.config.priority is not None: + self.headers["X-Priority"] = self.config.priority.value + self.max_concurrent_execution = self.config.max_concurrent_execution self.update_group_context = self.config.update_group_context @@ -217,6 +242,37 @@ def _echo(self, url: str, query: str, variables: dict | None = None) -> None: if variables: print(f"VARIABLES:\n{ujson.dumps(variables, indent=4)}\n") + def _request_headers(self, tracker: str | None = None, priority: Priority | None = None) -> dict: + """Build the per-request header delta to layer over the client's base headers. + + Returns only the request-specific entries (tracker, ``X-Priority``); the base headers + (auth, ``content-type``, and any client-wide default priority) are merged in per request + by the transport helpers. Keeping this a delta means the freshest ``self.headers`` — e.g. + an auth token refreshed during a relogin retry — always applies, while a caller can still + override any header, including auth, for a single request. + """ + headers: dict = {} + if self.insert_tracker and tracker: + headers["X-Infrahub-Tracker"] = tracker + effective_priority = priority + if effective_priority is None and self._request_context is not None: + effective_priority = self._request_context.priority + if effective_priority is not None: + headers["X-Priority"] = effective_priority.value + return headers + + def _merge_request_headers(self, headers: dict | None) -> dict: + """Merge a per-request header delta over the client's current base headers. + + Per-request entries take precedence over the client-wide base headers, so a caller may + override any header (including auth) for a single request, and a token refreshed mid-flight + during the automatic relogin retry is picked up from the freshly-copied ``self.headers``. + """ + merged = copy.copy(self.headers or {}) + if headers: + merged.update(headers) + return merged + @property def request_context(self) -> RequestContext | None: return self._request_context @@ -365,6 +421,15 @@ async def get_version(self) -> str: response = await self.execute_graphql(query="query { InfrahubInfo { version }}") return response.get("InfrahubInfo", {}).get("version", "") + async def get_server_information(self) -> ServerInfo: + """Return the Infrahub server information (version and deployment ID).""" + response = await self.execute_graphql( + query="query { InfrahubInfo { version deployment_id }}", + tracker="query-server-info", + ) + info = response.get("InfrahubInfo", {}) + return ServerInfo(version=info.get("version", ""), deployment_id=info.get("deployment_id", "")) + async def get_user(self) -> dict: """Return user information.""" return await self.execute_graphql(query=QUERY_USER, operation_name="GET_PROFILE_DETAILS") @@ -434,6 +499,7 @@ async def get( property: bool = ..., include_metadata: bool = ..., query_name: str | None = ..., + priority: Priority | None = ..., **kwargs: Any, ) -> SchemaType | None: ... @@ -455,6 +521,7 @@ async def get( property: bool = ..., include_metadata: bool = ..., query_name: str | None = ..., + priority: Priority | None = ..., **kwargs: Any, ) -> SchemaType: ... @@ -476,6 +543,7 @@ async def get( property: bool = ..., include_metadata: bool = ..., query_name: str | None = ..., + priority: Priority | None = ..., **kwargs: Any, ) -> SchemaType: ... @@ -497,6 +565,7 @@ async def get( property: bool = ..., include_metadata: bool = ..., query_name: str | None = ..., + priority: Priority | None = ..., **kwargs: Any, ) -> InfrahubNode | None: ... @@ -518,6 +587,7 @@ async def get( property: bool = ..., include_metadata: bool = ..., query_name: str | None = ..., + priority: Priority | None = ..., **kwargs: Any, ) -> InfrahubNode: ... @@ -539,6 +609,7 @@ async def get( property: bool = ..., include_metadata: bool = ..., query_name: str | None = ..., + priority: Priority | None = ..., **kwargs: Any, ) -> InfrahubNode: ... @@ -559,6 +630,7 @@ async def get( property: bool = False, include_metadata: bool = False, query_name: str | None = None, + priority: Priority | None = None, **kwargs: Any, ) -> InfrahubNode | SchemaType | None: branch = branch or self.default_branch @@ -596,6 +668,7 @@ async def get( property=property, include_metadata=include_metadata, query_name=query_name, + priority=priority, **filters, ) @@ -657,6 +730,7 @@ async def count( timeout: int | None = None, partial_match: bool = False, query_name: str | None = None, + priority: Priority | None = None, **kwargs: Any, ) -> int: """Return the number of nodes of a given kind.""" @@ -683,6 +757,7 @@ async def count( at=at, timeout=timeout, operation_name=query_name, + priority=priority, ) return int(response.get(schema.kind, {}).get("count", 0)) @@ -898,6 +973,7 @@ async def all( order: Order | None = ..., include_metadata: bool = ..., query_name: str | None = ..., + priority: Priority | None = ..., ) -> list[SchemaType]: ... @overload @@ -919,6 +995,7 @@ async def all( order: Order | None = ..., include_metadata: bool = ..., query_name: str | None = ..., + priority: Priority | None = ..., ) -> list[InfrahubNode]: ... async def all( @@ -939,6 +1016,7 @@ async def all( order: Order | None = None, include_metadata: bool = False, query_name: str | None = None, + priority: Priority | None = None, ) -> list[InfrahubNode] | list[SchemaType]: """Retrieve all nodes of a given kind. @@ -958,6 +1036,8 @@ async def all( order (Order, optional): Ordering related options. Setting `disable=True` enhances performances. include_metadata (bool, optional): If True, includes node_metadata and relationship_metadata in the query. query_name (str, optional): If provided is used as the GraphQL operation name else All_ is used. + priority (Priority, optional): Per-request priority emitted as the X-Priority header, overriding the + client default for these requests only. When None, the client default (if any) is used. Returns: list[InfrahubNode]: List of Nodes @@ -982,6 +1062,7 @@ async def all( order=order, include_metadata=include_metadata, query_name=query_name, + priority=priority, ) @overload @@ -1004,6 +1085,7 @@ async def filters( order: Order | None = ..., include_metadata: bool = ..., query_name: str | None = ..., + priority: Priority | None = ..., **kwargs: Any, ) -> list[SchemaType]: ... @@ -1027,6 +1109,7 @@ async def filters( order: Order | None = ..., include_metadata: bool = ..., query_name: str | None = ..., + priority: Priority | None = ..., **kwargs: Any, ) -> list[InfrahubNode]: ... @@ -1049,6 +1132,7 @@ async def filters( # noqa: C901 order: Order | None = None, include_metadata: bool = False, query_name: str | None = None, + priority: Priority | None = None, **kwargs: Any, ) -> list[InfrahubNode] | list[SchemaType]: """Retrieve nodes of a given kind based on provided filters. @@ -1070,6 +1154,8 @@ async def filters( # noqa: C901 order (Order, optional): Ordering related options. Setting `disable=True` enhances performances. include_metadata (bool, optional): If True, includes node_metadata and relationship_metadata in the query. query_name (str, optional): If provided is used as the GraphQL operation name else Filters_ is used. + priority (Priority, optional): Per-request priority emitted as the X-Priority header, overriding the + client default for these requests only. When None, the client default (if any) is used. **kwargs (Any): Additional filter criteria for the query. Returns: @@ -1086,29 +1172,38 @@ async def filters( # noqa: C901 filters = kwargs pagination_size = self.pagination_size + # Pagination is passed as GraphQL variables so the rendered query text stays + # identical across pages and can hit the server-side query cache. + query_data = await InfrahubNode(client=self, schema=schema, branch=branch).generate_query_data( + offset="$offset", + limit="$limit", + filters=filters, + include=include, + exclude=exclude, + fragment=fragment, + prefetch_relationships=prefetch_relationships, + partial_match=partial_match, + property=property, + order=order, + include_metadata=include_metadata, + ) + query = Query(query=query_data, name=query_name, variables={"offset": int, "limit": int}) + query_str = query.render() + async def process_page(page_offset: int, page_number: int) -> tuple[dict, ProcessRelationsNode]: """Process a single page of results.""" - query_data = await InfrahubNode(client=self, schema=schema, branch=branch).generate_query_data( - offset=page_offset if offset is None else offset, - limit=limit or pagination_size, - filters=filters, - include=include, - exclude=exclude, - fragment=fragment, - prefetch_relationships=prefetch_relationships, - partial_match=partial_match, - property=property, - order=order, - include_metadata=include_metadata, - ) - query = Query(query=query_data, name=query_name) response = await self.execute_graphql( - query=query.render(), + query=query_str, + variables={ + "offset": page_offset if offset is None else offset, + "limit": limit or pagination_size, + }, branch_name=branch, at=at, tracker=f"query-{str(schema.kind).lower()}-page{page_number}", timeout=timeout, operation_name=query.name, + priority=priority, ) process_result: ProcessRelationsNode = await self._process_nodes_and_relationships( @@ -1126,7 +1221,9 @@ async def process_batch() -> tuple[list[InfrahubNode], list[InfrahubNode]]: nodes = [] related_nodes = [] batch_process = await self.create_batch() - count = await self.count(kind=schema.kind, branch=branch, partial_match=partial_match, **filters) + count = await self.count( + kind=schema.kind, branch=branch, partial_match=partial_match, priority=priority, **filters + ) total_pages = (count + pagination_size - 1) // pagination_size for page_number in range(1, total_pages + 1): @@ -1185,6 +1282,7 @@ async def execute_graphql( timeout: int | None = None, tracker: str | None = None, operation_name: str | None = None, + priority: Priority | None = None, ) -> dict: """Execute a GraphQL query (or mutation). @@ -1198,6 +1296,8 @@ async def execute_graphql( timeout (int, optional): Timeout in second for the query. Defaults to None. operation_name (str, optional): GraphQL operation name, sent as `operationName` in the request payload so tracing/observability tools can identify the operation. Defaults to None. + priority (Priority, optional): Per-request priority emitted as the X-Priority header. Overrides the + client-wide default for this request only. When None, the client default (if any) is used. Returns: dict: The GraphQL data payload (response["data"]). @@ -1219,9 +1319,7 @@ async def execute_graphql( if operation_name: payload["operationName"] = operation_name - headers = copy.copy(self.headers or {}) - if self.insert_tracker and tracker: - headers["X-Infrahub-Tracker"] = tracker + headers = self._request_headers(tracker=tracker, priority=priority) self._echo(url=url, query=query, variables=variables) @@ -1275,6 +1373,7 @@ async def _execute_graphql_with_file( timeout: int | None = None, tracker: str | None = None, operation_name: str | None = None, + priority: Priority | None = None, ) -> dict: """Execute a GraphQL mutation with a file upload using multipart/form-data. @@ -1289,6 +1388,8 @@ async def _execute_graphql_with_file( branch_name: Name of the branch on which the mutation will be executed. timeout: Timeout in seconds for the query. tracker: Optional tracker for request tracing. + priority: Per-request priority emitted as the X-Priority header, overriding the client + default for this request only. When None, the client default (if any) is used. Returns: dict: The GraphQL data payload (response["data"]). @@ -1304,11 +1405,9 @@ async def _execute_graphql_with_file( variables = variables or {} variables["file"] = None - headers = copy.copy(self.headers or {}) - # Remove content-type header - httpx will set it for multipart - headers.pop("content-type", None) - if self.insert_tracker and tracker: - headers["X-Infrahub-Tracker"] = tracker + # content-type is popped from the base headers by _post_multipart (httpx sets the + # multipart boundary itself); only the request-specific delta is built here. + headers = self._request_headers(tracker=tracker, priority=priority) self._echo(url=url, query=query, variables=variables) @@ -1349,11 +1448,9 @@ async def _post_multipart( """ await self.login() - headers = headers or {} - base_headers = copy.copy(self.headers or {}) - # Remove content-type from base headers - httpx will set it for multipart - base_headers.pop("content-type", None) - headers.update(base_headers) + headers = self._merge_request_headers(headers) + # Remove content-type - httpx sets it (with the multipart boundary) itself + headers.pop("content-type", None) # Build the multipart form data according to GraphQL Multipart Request Spec files = MultipartBuilder.build_payload( @@ -1390,14 +1487,18 @@ async def _request_multipart( ServerNotResponsiveError: If the server didn't respond before the timeout expired. """ - async with httpx.AsyncClient(**self._build_proxy_config(), verify=self.config.tls_context) as client: - try: - response = await client.post(url=url, headers=headers, timeout=timeout, files=files) - except httpx.NetworkError as exc: - raise ServerNotReachableError(address=self.address) from exc - except httpx.ReadTimeout as exc: - raise ServerNotResponsiveError(url=url, timeout=timeout) from exc + async def send() -> httpx.Response: + _rewind_multipart_files(files) + async with httpx.AsyncClient(**self._build_proxy_config(), verify=self.config.tls_context) as client: + try: + return await client.post(url=url, headers=headers, timeout=timeout, files=files) + except httpx.NetworkError as exc: + raise ServerNotReachableError(address=self.address) from exc + except httpx.ReadTimeout as exc: + raise ServerNotResponsiveError(url=url, timeout=timeout) from exc + + response = await self._rate_limit_handler.asend(send=send, url=url) self._record(response) return response @@ -1418,9 +1519,7 @@ async def _post( """ await self.login() - headers = headers or {} - base_headers = copy.copy(self.headers or {}) - headers.update(base_headers) + headers = self._merge_request_headers(headers) return await self._request( url=url, @@ -1441,9 +1540,7 @@ async def _get(self, url: str, headers: dict | None = None, timeout: int | None """ await self.login() - headers = headers or {} - base_headers = copy.copy(self.headers or {}) - headers.update(base_headers) + headers = self._merge_request_headers(headers) return await self._request( url=url, @@ -1461,6 +1558,9 @@ async def _get_streaming( Returns an async context manager that yields the streaming response. Use this for downloading large files without loading into memory. + Yields: + httpx.Response: The streaming HTTP response. + Raises: ServerNotReachableError: If we are not able to connect to the server. ServerNotResponsiveError: If the server didn't respond before the timeout expired. @@ -1468,20 +1568,41 @@ async def _get_streaming( """ await self.login() - headers = headers or {} - base_headers = copy.copy(self.headers or {}) - headers.update(base_headers) + headers = self._merge_request_headers(headers) + request_timeout = timeout or self.default_timeout async with httpx.AsyncClient(**self._build_proxy_config(), verify=self.config.tls_context) as client: + open_stream: dict[str, AsyncExitStack] = {} + + async def send() -> httpx.Response: + # Retry stream initiation only (a 429 arrives in the headers before the body): a + # failed attempt is read and closed here, a successful stream is left open for + # the caller and closed afterwards. + stack = AsyncExitStack() + response = await stack.enter_async_context( + client.stream(method="GET", url=url, headers=headers, timeout=request_timeout) + ) + if response.status_code == 429: + try: + await response.aread() + finally: + await stack.aclose() + else: + open_stream["stack"] = stack + return response + try: - async with client.stream( - method="GET", url=url, headers=headers, timeout=timeout or self.default_timeout - ) as response: + response = await self._rate_limit_handler.asend(send=send, url=url) + try: yield response + finally: + stack = open_stream.get("stack") + if stack is not None: + await stack.aclose() except httpx.NetworkError as exc: raise ServerNotReachableError(address=self.address) from exc except httpx.ReadTimeout as exc: - raise ServerNotResponsiveError(url=url, timeout=timeout or self.default_timeout) from exc + raise ServerNotResponsiveError(url=url, timeout=request_timeout) from exc async def _request( self, @@ -1491,7 +1612,10 @@ async def _request( timeout: int, payload: dict | None = None, ) -> httpx.Response: - response = await self._request_method(url=url, method=method, headers=headers, timeout=timeout, payload=payload) + async def send() -> httpx.Response: + return await self._request_method(url=url, method=method, headers=headers, timeout=timeout, payload=payload) + + response = await self._rate_limit_handler.asend(send=send, url=url) self._record(response) return response @@ -1597,10 +1721,7 @@ async def query_gql_query( url_params = copy.deepcopy(params or {}) url_params["branch"] = branch_name or self.default_branch - headers = copy.copy(self.headers or {}) - - if self.insert_tracker and tracker: - headers["X-Infrahub-Tracker"] = tracker + headers = self._request_headers(tracker=tracker) if at: url_params["at"] = at @@ -1647,6 +1768,7 @@ async def create_diff( from_time: datetime, to_time: datetime, wait_until_completion: bool = True, + priority: Priority | None = None, ) -> bool | str: if from_time > to_time: raise ValueError("from_time must be <= to_time") @@ -1662,7 +1784,7 @@ async def create_diff( mutation_query = MUTATION_QUERY_TASK if not wait_until_completion else {"ok": None} query = Mutation(mutation="DiffUpdate", input_data=input_data, query=mutation_query) - response = await self.execute_graphql(query=query.render(), tracker="mutation-diff-update") + response = await self.execute_graphql(query=query.render(), tracker="mutation-diff-update", priority=priority) if not wait_until_completion and "task" in response["DiffUpdate"]: return response["DiffUpdate"]["task"]["id"] @@ -1677,6 +1799,7 @@ async def get_diff_summary( to_time: datetime | None = None, timeout: int | None = None, tracker: str | None = None, + priority: Priority | None = None, ) -> list[NodeDiff]: query = get_diff_summary_query() input_data = {"branch_name": branch} @@ -1695,6 +1818,7 @@ async def get_diff_summary( tracker=tracker, variables=input_data, operation_name="GetDiffTree", + priority=priority, ) node_diffs: list[NodeDiff] = [] @@ -1716,6 +1840,7 @@ async def get_diff_tree( to_time: datetime | None = None, timeout: int | None = None, tracker: str | None = None, + priority: Priority | None = None, ) -> DiffTreeData | None: """Get complete diff tree with metadata and nodes. @@ -1743,6 +1868,7 @@ async def get_diff_tree( tracker=tracker, variables=input_data, operation_name=query.name, + priority=priority, ) diff_tree = response["DiffTree"] @@ -2072,6 +2198,15 @@ def get_version(self) -> str: response = self.execute_graphql(query="query { InfrahubInfo { version }}") return response.get("InfrahubInfo", {}).get("version", "") + def get_server_information(self) -> ServerInfo: + """Return the Infrahub server information (version and deployment ID).""" + response = self.execute_graphql( + query="query { InfrahubInfo { version deployment_id }}", + tracker="query-server-info", + ) + info = response.get("InfrahubInfo", {}) + return ServerInfo(version=info.get("version", ""), deployment_id=info.get("deployment_id", "")) + def get_user(self) -> dict: """Return user information.""" return self.execute_graphql(query=QUERY_USER, operation_name="GET_PROFILE_DETAILS") @@ -2135,6 +2270,7 @@ def execute_graphql( timeout: int | None = None, tracker: str | None = None, operation_name: str | None = None, + priority: Priority | None = None, ) -> dict: """Execute a GraphQL query (or mutation). @@ -2148,6 +2284,8 @@ def execute_graphql( timeout (int, optional): Timeout in second for the query. Defaults to None. operation_name (str, optional): GraphQL operation name, sent as `operationName` in the request payload so tracing/observability tools can identify the operation. Defaults to None. + priority (Priority, optional): Per-request priority emitted as the X-Priority header. Overrides the + client-wide default for this request only. When None, the client default (if any) is used. Returns: dict: The GraphQL data payload (`response["data"]`). @@ -2169,9 +2307,7 @@ def execute_graphql( if operation_name: payload["operationName"] = operation_name - headers = copy.copy(self.headers or {}) - if self.insert_tracker and tracker: - headers["X-Infrahub-Tracker"] = tracker + headers = self._request_headers(tracker=tracker, priority=priority) self._echo(url=url, query=query, variables=variables) @@ -2225,6 +2361,7 @@ def _execute_graphql_with_file( timeout: int | None = None, tracker: str | None = None, operation_name: str | None = None, + priority: Priority | None = None, ) -> dict: """Execute a GraphQL mutation with a file upload using multipart/form-data. @@ -2239,6 +2376,8 @@ def _execute_graphql_with_file( branch_name: Name of the branch on which the mutation will be executed. timeout: Timeout in seconds for the query. tracker: Optional tracker for request tracing. + priority: Per-request priority emitted as the X-Priority header, overriding the client + default for this request only. When None, the client default (if any) is used. Returns: dict: The GraphQL data payload (response["data"]). @@ -2254,11 +2393,9 @@ def _execute_graphql_with_file( variables = variables or {} variables["file"] = None - headers = copy.copy(self.headers or {}) - # Remove content-type header - httpx will set it for multipart - headers.pop("content-type", None) - if self.insert_tracker and tracker: - headers["X-Infrahub-Tracker"] = tracker + # content-type is popped from the base headers by _post_multipart (httpx sets the + # multipart boundary itself); only the request-specific delta is built here. + headers = self._request_headers(tracker=tracker, priority=priority) self._echo(url=url, query=query, variables=variables) @@ -2299,11 +2436,9 @@ def _post_multipart( """ self.login() - headers = headers or {} - base_headers = copy.copy(self.headers or {}) - # Remove content-type from base headers - httpx will set it for multipart - base_headers.pop("content-type", None) - headers.update(base_headers) + headers = self._merge_request_headers(headers) + # Remove content-type - httpx sets it (with the multipart boundary) itself + headers.pop("content-type", None) # Build the multipart form data according to GraphQL Multipart Request Spec files = MultipartBuilder.build_payload( @@ -2338,14 +2473,18 @@ def _request_multipart( ServerNotResponsiveError: If the server didn't respond before the timeout expired. """ - with httpx.Client(**self._build_proxy_config(), verify=self.config.tls_context) as client: - try: - response = client.post(url=url, headers=headers, timeout=timeout, files=files) - except httpx.NetworkError as exc: - raise ServerNotReachableError(address=self.address) from exc - except httpx.ReadTimeout as exc: - raise ServerNotResponsiveError(url=url, timeout=timeout) from exc + def send() -> httpx.Response: + _rewind_multipart_files(files) + with httpx.Client(**self._build_proxy_config(), verify=self.config.tls_context) as client: + try: + return client.post(url=url, headers=headers, timeout=timeout, files=files) + except httpx.NetworkError as exc: + raise ServerNotReachableError(address=self.address) from exc + except httpx.ReadTimeout as exc: + raise ServerNotResponsiveError(url=url, timeout=timeout) from exc + + response = self._rate_limit_handler.send(send=send, url=url) self._record(response) return response @@ -2357,6 +2496,7 @@ def count( timeout: int | None = None, partial_match: bool = False, query_name: str | None = None, + priority: Priority | None = None, **kwargs: Any, ) -> int: """Return the number of nodes of a given kind.""" @@ -2383,6 +2523,7 @@ def count( at=at, timeout=timeout, operation_name=query_name, + priority=priority, ) return int(response.get(schema.kind, {}).get("count", 0)) @@ -2598,6 +2739,7 @@ def all( order: Order | None = ..., include_metadata: bool = ..., query_name: str | None = ..., + priority: Priority | None = ..., ) -> list[SchemaTypeSync]: ... @overload @@ -2619,6 +2761,7 @@ def all( order: Order | None = ..., include_metadata: bool = ..., query_name: str | None = ..., + priority: Priority | None = ..., ) -> list[InfrahubNodeSync]: ... def all( @@ -2639,6 +2782,7 @@ def all( order: Order | None = None, include_metadata: bool = False, query_name: str | None = None, + priority: Priority | None = None, ) -> list[InfrahubNodeSync] | list[SchemaTypeSync]: """Retrieve all nodes of a given kind. @@ -2658,6 +2802,8 @@ def all( order (Order, optional): Ordering related options. Setting `disable=True` enhances performances. include_metadata (bool, optional): If True, includes node_metadata and relationship_metadata in the query. query_name (str, optional): If provided is used as the GraphQL operation name else All_ is used. + priority (Priority, optional): Per-request priority emitted as the X-Priority header, overriding the + client default for these requests only. When None, the client default (if any) is used. Returns: list[InfrahubNodeSync]: List of Nodes @@ -2682,6 +2828,7 @@ def all( order=order, include_metadata=include_metadata, query_name=query_name, + priority=priority, ) def _process_nodes_and_relationships( @@ -2745,6 +2892,7 @@ def filters( order: Order | None = ..., include_metadata: bool = ..., query_name: str | None = ..., + priority: Priority | None = ..., **kwargs: Any, ) -> list[SchemaTypeSync]: ... @@ -2768,6 +2916,7 @@ def filters( order: Order | None = ..., include_metadata: bool = ..., query_name: str | None = ..., + priority: Priority | None = ..., **kwargs: Any, ) -> list[InfrahubNodeSync]: ... @@ -2790,6 +2939,7 @@ def filters( # noqa: C901 order: Order | None = None, include_metadata: bool = False, query_name: str | None = None, + priority: Priority | None = None, **kwargs: Any, ) -> list[InfrahubNodeSync] | list[SchemaTypeSync]: """Retrieve nodes of a given kind based on provided filters. @@ -2811,6 +2961,8 @@ def filters( # noqa: C901 order (Order, optional): Ordering related options. Setting `disable=True` enhances performances. include_metadata (bool, optional): If True, includes node_metadata and relationship_metadata in the query. query_name (str, optional): If provided is used as the GraphQL operation name else Filters_ is used. + priority (Priority, optional): Per-request priority emitted as the X-Priority header, overriding the + client default for these requests only. When None, the client default (if any) is used. **kwargs (Any): Additional filter criteria for the query. Returns: @@ -2827,29 +2979,38 @@ def filters( # noqa: C901 filters = kwargs pagination_size = self.pagination_size + # Pagination is passed as GraphQL variables so the rendered query text stays + # identical across pages and can hit the server-side query cache. + query_data = InfrahubNodeSync(client=self, schema=schema, branch=branch).generate_query_data( + offset="$offset", + limit="$limit", + filters=filters, + include=include, + exclude=exclude, + fragment=fragment, + prefetch_relationships=prefetch_relationships, + partial_match=partial_match, + property=property, + order=order, + include_metadata=include_metadata, + ) + query = Query(query=query_data, name=query_name, variables={"offset": int, "limit": int}) + query_str = query.render() + def process_page(page_offset: int, page_number: int) -> tuple[dict, ProcessRelationsNodeSync]: """Process a single page of results.""" - query_data = InfrahubNodeSync(client=self, schema=schema, branch=branch).generate_query_data( - offset=page_offset if offset is None else offset, - limit=limit or pagination_size, - filters=filters, - include=include, - exclude=exclude, - fragment=fragment, - prefetch_relationships=prefetch_relationships, - partial_match=partial_match, - property=property, - order=order, - include_metadata=include_metadata, - ) - query = Query(query=query_data, name=query_name) response = self.execute_graphql( - query=query.render(), + query=query_str, + variables={ + "offset": page_offset if offset is None else offset, + "limit": limit or pagination_size, + }, branch_name=branch, at=at, timeout=timeout, tracker=f"query-{str(schema.kind).lower()}-page{page_number}", operation_name=query.name, + priority=priority, ) process_result: ProcessRelationsNodeSync = self._process_nodes_and_relationships( @@ -2868,7 +3029,9 @@ def process_batch() -> tuple[list[InfrahubNodeSync], list[InfrahubNodeSync]]: related_nodes = [] batch_process = self.create_batch() - count = self.count(kind=schema.kind, branch=branch, partial_match=partial_match, **filters) + count = self.count( + kind=schema.kind, branch=branch, partial_match=partial_match, priority=priority, **filters + ) total_pages = (count + pagination_size - 1) // pagination_size for page_number in range(1, total_pages + 1): @@ -2933,6 +3096,7 @@ def get( property: bool = ..., include_metadata: bool = ..., query_name: str | None = ..., + priority: Priority | None = ..., **kwargs: Any, ) -> SchemaTypeSync | None: ... @@ -2954,6 +3118,7 @@ def get( property: bool = ..., include_metadata: bool = ..., query_name: str | None = ..., + priority: Priority | None = ..., **kwargs: Any, ) -> SchemaTypeSync: ... @@ -2975,6 +3140,7 @@ def get( property: bool = ..., include_metadata: bool = ..., query_name: str | None = ..., + priority: Priority | None = ..., **kwargs: Any, ) -> SchemaTypeSync: ... @@ -2996,6 +3162,7 @@ def get( property: bool = ..., include_metadata: bool = ..., query_name: str | None = ..., + priority: Priority | None = ..., **kwargs: Any, ) -> InfrahubNodeSync | None: ... @@ -3017,6 +3184,7 @@ def get( property: bool = ..., include_metadata: bool = ..., query_name: str | None = ..., + priority: Priority | None = ..., **kwargs: Any, ) -> InfrahubNodeSync: ... @@ -3038,6 +3206,7 @@ def get( property: bool = ..., include_metadata: bool = ..., query_name: str | None = ..., + priority: Priority | None = ..., **kwargs: Any, ) -> InfrahubNodeSync: ... @@ -3058,6 +3227,7 @@ def get( property: bool = False, include_metadata: bool = False, query_name: str | None = None, + priority: Priority | None = None, **kwargs: Any, ) -> InfrahubNodeSync | SchemaTypeSync | None: branch = branch or self.default_branch @@ -3095,6 +3265,7 @@ def get( property=property, include_metadata=include_metadata, query_name=query_name, + priority=priority, **filters, ) @@ -3143,10 +3314,7 @@ def query_gql_query( url_params = copy.deepcopy(params or {}) url_params["branch"] = branch_name or self.default_branch - headers = copy.copy(self.headers or {}) - - if self.insert_tracker and tracker: - headers["X-Infrahub-Tracker"] = tracker + headers = self._request_headers(tracker=tracker) if at: url_params["at"] = at @@ -3192,6 +3360,7 @@ def create_diff( from_time: datetime, to_time: datetime, wait_until_completion: bool = True, + priority: Priority | None = None, ) -> bool | str: if from_time > to_time: raise ValueError("from_time must be <= to_time") @@ -3207,7 +3376,7 @@ def create_diff( mutation_query = MUTATION_QUERY_TASK if not wait_until_completion else {"ok": None} query = Mutation(mutation="DiffUpdate", input_data=input_data, query=mutation_query) - response = self.execute_graphql(query=query.render(), tracker="mutation-diff-update") + response = self.execute_graphql(query=query.render(), tracker="mutation-diff-update", priority=priority) if not wait_until_completion and "task" in response["DiffUpdate"]: return response["DiffUpdate"]["task"]["id"] @@ -3222,6 +3391,7 @@ def get_diff_summary( to_time: datetime | None = None, timeout: int | None = None, tracker: str | None = None, + priority: Priority | None = None, ) -> list[NodeDiff]: query = get_diff_summary_query() input_data = {"branch_name": branch} @@ -3240,6 +3410,7 @@ def get_diff_summary( tracker=tracker, variables=input_data, operation_name="GetDiffTree", + priority=priority, ) node_diffs: list[NodeDiff] = [] @@ -3261,6 +3432,7 @@ def get_diff_tree( to_time: datetime | None = None, timeout: int | None = None, tracker: str | None = None, + priority: Priority | None = None, ) -> DiffTreeData | None: """Get complete diff tree with metadata and nodes. @@ -3288,6 +3460,7 @@ def get_diff_tree( tracker=tracker, variables=input_data, operation_name=query.name, + priority=priority, ) diff_tree = response["DiffTree"] @@ -3446,7 +3619,7 @@ def allocate_next_ip_prefix( Args: resource_pool (InfrahubNodeSync): Node corresponding to the pool to allocate resources from. identifier (str, optional): Value to perform idempotent allocation, the same resource will be returned for a given identifier. - size (int, optional): Length of the prefix to allocate. + prefix_length (int, optional): Length of the prefix to allocate. member_type (str, optional): Member type of the prefix to allocate. prefix_type (str, optional): Kind of the prefix to allocate. data (dict, optional): A key/value map to use to set attributes values on the allocated prefix. @@ -3509,9 +3682,7 @@ def _get(self, url: str, headers: dict | None = None, timeout: int | None = None """ self.login() - headers = headers or {} - base_headers = copy.copy(self.headers or {}) - headers.update(base_headers) + headers = self._merge_request_headers(headers) return self._request( url=url, @@ -3529,6 +3700,9 @@ def _get_streaming( Returns a context manager that yields the streaming response. Use this for downloading large files without loading into memory. + Yields: + httpx.Response: The streaming HTTP response. + Raises: ServerNotReachableError: If we are not able to connect to the server. ServerNotResponsiveError: If the server didn't respond before the timeout expired. @@ -3536,20 +3710,41 @@ def _get_streaming( """ self.login() - headers = headers or {} - base_headers = copy.copy(self.headers or {}) - headers.update(base_headers) + headers = self._merge_request_headers(headers) + request_timeout = timeout or self.default_timeout with httpx.Client(**self._build_proxy_config(), verify=self.config.tls_context) as client: + open_stream: dict[str, ExitStack] = {} + + def send() -> httpx.Response: + # Retry stream initiation only (a 429 arrives in the headers before the body): a + # failed attempt is read and closed here, a successful stream is left open for + # the caller and closed afterwards. + stack = ExitStack() + response = stack.enter_context( + client.stream(method="GET", url=url, headers=headers, timeout=request_timeout) + ) + if response.status_code == 429: + try: + response.read() + finally: + stack.close() + else: + open_stream["stack"] = stack + return response + try: - with client.stream( - method="GET", url=url, headers=headers, timeout=timeout or self.default_timeout - ) as response: + response = self._rate_limit_handler.send(send=send, url=url) + try: yield response + finally: + stack = open_stream.get("stack") + if stack is not None: + stack.close() except httpx.NetworkError as exc: raise ServerNotReachableError(address=self.address) from exc except httpx.ReadTimeout as exc: - raise ServerNotResponsiveError(url=url, timeout=timeout or self.default_timeout) from exc + raise ServerNotResponsiveError(url=url, timeout=request_timeout) from exc @handle_relogin_sync def _post( @@ -3568,9 +3763,7 @@ def _post( """ self.login() - headers = headers or {} - base_headers = copy.copy(self.headers or {}) - headers.update(base_headers) + headers = self._merge_request_headers(headers) return self._request( url=url, @@ -3588,7 +3781,10 @@ def _request( timeout: int, payload: dict | None = None, ) -> httpx.Response: - response = self._request_method(url=url, method=method, headers=headers, timeout=timeout, payload=payload) + def send() -> httpx.Response: + return self._request_method(url=url, method=method, headers=headers, timeout=timeout, payload=payload) + + response = self._rate_limit_handler.send(send=send, url=url) self._record(response) return response diff --git a/infrahub_sdk/config.py b/infrahub_sdk/config.py index e9e66e6f4..f81fc911e 100644 --- a/infrahub_sdk/config.py +++ b/infrahub_sdk/config.py @@ -8,7 +8,7 @@ from pydantic_settings import BaseSettings, InitSettingsSource, PydanticBaseSettingsSource, SettingsConfigDict from typing_extensions import Self -from .constants import InfrahubClientMode +from .constants import InfrahubClientMode, Priority from .playback import JSONPlayback from .recorder import JSONRecorder, NoRecorder, Recorder, RecorderType from .types import AsyncRequester, InfrahubLoggers, RequesterTransport, SyncRequester @@ -56,8 +56,34 @@ class ConfigBase(BaseSettings): max_concurrent_execution: int = Field(default=5, description="Max concurrent execution in batch mode") mode: InfrahubClientMode = Field(default=InfrahubClientMode.DEFAULT, description="Default mode for the client") pagination_size: int = Field(default=50, description="Page size for queries to the server") + priority: Priority | None = Field( + default=None, + description=( + "Default request priority emitted as the X-Priority header on every request; " + "one of high|medium|low (case-insensitive). When unset, no header is sent." + ), + ) retry_delay: int = Field(default=5, description="Number of seconds to wait until attempting a retry.") retry_on_failure: bool = Field(default=False, description="Retry operation in case of failure") + rate_limit_retry_enabled: bool = Field( + default=True, + description="Retry requests that receive HTTP 429 using backoff. Set False to disable.", + ) + rate_limit_max_retries: int = Field( + default=10, + ge=0, + description="Maximum number of retries after the initial attempt when receiving HTTP 429.", + ) + rate_limit_backoff_base: float = Field( + default=0.5, + gt=0, + description="Base interval in seconds for exponential backoff between 429 retries.", + ) + rate_limit_backoff_max: float = Field( + default=60.0, + gt=0, + description="Maximum wait in seconds for any single 429 retry (also clamps Retry-After).", + ) max_retry_duration: int = Field( default=300, description="Maximum duration until we stop attempting to retry if enabled." ) diff --git a/infrahub_sdk/constants.py b/infrahub_sdk/constants.py index 04dd6b955..785445d9a 100644 --- a/infrahub_sdk/constants.py +++ b/infrahub_sdk/constants.py @@ -1,3 +1,5 @@ +"""Enumerations shared across the Infrahub SDK.""" + import enum @@ -5,3 +7,23 @@ class InfrahubClientMode(str, enum.Enum): DEFAULT = "default" TRACKING = "tracking" # IDEMPOTENT = "idempotent" + + +class Priority(str, enum.Enum): + """Request priority emitted as the ``X-Priority`` header. + + String-valued closed enum matched case-insensitively (e.g. "LOW", "Low" and "low" all + resolve to :attr:`Priority.LOW`). + """ + + HIGH = "high" + MEDIUM = "medium" + LOW = "low" + + @classmethod + def _missing_(cls, value: object) -> "Priority | None": + if isinstance(value, str): + for member in cls: + if member.value == value.lower(): + return member + return None diff --git a/infrahub_sdk/context.py b/infrahub_sdk/context.py index 201a9ef98..38d4daeb0 100644 --- a/infrahub_sdk/context.py +++ b/infrahub_sdk/context.py @@ -2,6 +2,8 @@ from pydantic import BaseModel, Field +from .constants import Priority # noqa: TC001 (pydantic needs this at runtime to build the model schema) + class ContextAccount(BaseModel): id: str = Field(..., description="The ID of the account") @@ -11,3 +13,6 @@ class RequestContext(BaseModel): """The context can be used to override settings such as the account within mutations.""" account: ContextAccount | None = Field(default=None, description="Account tied to the context") + priority: Priority | None = Field( + default=None, description="Request priority emitted as the X-Priority header (not part of the mutation body)" + ) diff --git a/infrahub_sdk/ctl/cli_commands.py b/infrahub_sdk/ctl/cli_commands.py index b3768f5cd..9d947ff1f 100644 --- a/infrahub_sdk/ctl/cli_commands.py +++ b/infrahub_sdk/ctl/cli_commands.py @@ -414,6 +414,7 @@ def info( # noqa: PLR0915 "error": None, "status": ":x:", "infrahub_version": "N/A", + "deployment_id": "N/A", "user_info": {}, "groups": {}, } @@ -421,7 +422,9 @@ def info( # noqa: PLR0915 fetch_user_details = bool(client.config.username) or bool(client.config.api_token) try: - info["infrahub_version"] = client.get_version() + server_info = client.get_server_information() + info["infrahub_version"] = server_info.version + info["deployment_id"] = server_info.deployment_id if fetch_user_details: info["user_info"] = client.get_user() @@ -467,6 +470,7 @@ def info( # noqa: PLR0915 version_info = Table(show_header=False, box=None) version_info.add_row("Python Version:", platform.python_version()) version_info.add_row("Infrahub Version", info["infrahub_version"]) + version_info.add_row("Deployment ID:", info["deployment_id"]) version_info.add_row("Infrahub SDK:", sdk_version) layout["version_info"].update(Panel(version_info, title="Version Information")) @@ -509,6 +513,7 @@ def info( # noqa: PLR0915 table.add_row("Python Version:", platform.python_version()) table.add_row("SDK Version:", sdk_version) table.add_row("Infrahub Version:", info["infrahub_version"]) + table.add_row("Deployment ID:", info["deployment_id"]) if account := info["user_info"].get("AccountProfile"): table.add_row("User:", account["display_label"]) diff --git a/infrahub_sdk/ctl/config.py b/infrahub_sdk/ctl/config.py index 03ca58ca1..4ada5d555 100644 --- a/infrahub_sdk/ctl/config.py +++ b/infrahub_sdk/ctl/config.py @@ -81,8 +81,8 @@ def load_and_exit(self, config_file: str | Path = "infrahubctl.toml", config_dat In such cases, a message is printed to the screen indicating the settings which don't pass validation. Args: - config_file_name (str, optional): [description]. Defaults to "pyprojectctl.toml". - config_data (dict, optional): [description]. Defaults to None. + config_file (str | Path, optional): Path to the configuration file. Defaults to "infrahubctl.toml". + config_data (dict, optional): In-memory configuration overriding the file contents. Defaults to None. """ try: diff --git a/infrahub_sdk/ctl/schema.py b/infrahub_sdk/ctl/schema.py index f3d42afea..46b7f2072 100644 --- a/infrahub_sdk/ctl/schema.py +++ b/infrahub_sdk/ctl/schema.py @@ -1,51 +1,70 @@ from __future__ import annotations import asyncio +import difflib import time from datetime import datetime, timezone +from enum import Enum from pathlib import Path -from typing import TYPE_CHECKING, Any, Literal +from typing import Any, Literal import typer import yaml -from pydantic import ValidationError from rich.console import Console +from rich.markup import escape from rich.table import Table from ..async_typer import AsyncTyper from ..ctl.client import initialize_client from ..ctl.utils import catch_exception, init_logging from ..queries import SCHEMA_HASH_SYNC_STATUS -from ..schema import NodeSchemaAPI, SchemaWarning +from ..schema import NodeSchemaAPI, SchemaWarning, validate_schema from ..yaml import SchemaFile from .parameters import CONFIG_PARAM +from .schema_format import ( + DEFAULT_BACKFILL_ORDER_WEIGHT, + FormatError, + FormatOptions, + format_schema_text, + is_schema_document, +) from .utils import load_yamlfile_from_disk_and_exit -if TYPE_CHECKING: - from .. import InfrahubClient - SchemaContainer = Literal["nodes", "generics", "relationships"] app = AsyncTyper() console = Console() +class FormatOutcome(Enum): + """Result of formatting a single schema file.""" + + ERROR = "error" + SKIPPED = "skipped" + UNCHANGED = "unchanged" + CHANGED = "changed" + + @app.callback() def callback() -> None: """Manage the schema in a remote Infrahub instance.""" -def validate_schema_content_and_exit(client: InfrahubClient, schemas: list[SchemaFile]) -> None: +def validate_schema_content_and_exit(schemas: list[SchemaFile]) -> None: + """Report every offline contract violation and exit when at least one schema is invalid. + + Read-only fields are reported by the server on the load/check response, so only errors are + rendered here to avoid warning about the same field twice. + """ has_error: bool = False for schema_file in schemas: - try: - client.schema.validate(data=schema_file.payload) - except ValidationError as exc: - console.print(f"[red]Schema not valid, found '{len(exc.errors())}' error(s) in {schema_file.location}") - has_error = True - for error in exc.errors(): - loc_str = [str(item) for item in error["loc"]] - console.print(f" '{'/'.join(loc_str)}' | {error['msg']} ({error['type']})") + result = validate_schema(schema=schema_file.payload) + if result.valid: + continue + has_error = True + console.print(f"[red]Schema not valid, found '{len(result.errors)}' error(s) in {schema_file.location}") + for error in result.errors: + console.print(f" {escape(error.message)}") if has_error: raise typer.Exit(1) @@ -190,7 +209,7 @@ async def load( schemas_data = load_yamlfile_from_disk_and_exit(paths=schemas, file_type=SchemaFile, console=console) schema_definition = "schema" if len(schemas_data) == 1 else "schemas" client = initialize_client() - validate_schema_content_and_exit(client=client, schemas=schemas_data) + validate_schema_content_and_exit(schemas=schemas_data) start_time = time.time() response = await client.schema.load(schemas=[item.payload for item in schemas_data], branch=branch) @@ -240,7 +259,7 @@ async def check( schemas_data = load_yamlfile_from_disk_and_exit(paths=schemas, file_type=SchemaFile, console=console) client = initialize_client() - validate_schema_content_and_exit(client=client, schemas=schemas_data) + validate_schema_content_and_exit(schemas=schemas_data) success, response = await client.schema.check(schemas=[item.payload for item in schemas_data], branch=branch) @@ -262,9 +281,9 @@ async def check( def _display_schema_warnings(console: Console, warnings: list[SchemaWarning]) -> None: for warning in warnings: - console.print( - f"[yellow] {warning.type.value}: {warning.message} [{', '.join([kind.display for kind in warning.kinds])}]" - ) + # A warning about a top-level key has no kind to attribute it to. + kinds = f" [{', '.join(kind.display for kind in warning.kinds)}]" if warning.kinds else "" + console.print(f"[yellow] {warning.type.value}: {escape(warning.message)}{escape(kinds)}") def _default_export_directory() -> Path: @@ -414,3 +433,153 @@ async def schema_show( "Yes" if rel.optional else "No", ) console.print(rel_table) + + +def _print_schema_diff(location: Path, original: str, formatted: str) -> None: + diff = difflib.unified_diff( + original.splitlines(keepends=True), + formatted.splitlines(keepends=True), + fromfile=f"{location} (current)", + tofile=f"{location} (formatted)", + ) + for line in diff: + # markup=False keeps bracketed diff content (e.g. `[manufacturer, name]`) + # literal, so colour is applied via style= rather than inline markup. + if line.startswith("+") and not line.startswith("+++"): + console.print(line, end="", markup=False, highlight=False, style="green") + elif line.startswith("-") and not line.startswith("---"): + console.print(line, end="", markup=False, highlight=False, style="red") + else: + console.print(line, end="", markup=False, highlight=False) + + +def _format_one_schema_file( + location: Path, entries: list[SchemaFile], check: bool, diff: bool, options: FormatOptions +) -> FormatOutcome: + """Format a single schema file and report what happened. + + Args: + location: Path of the file on disk. + entries: SchemaFile entries parsed for this location (more than one means + a genuine multi-document file, which is not supported). + check: Report changes without writing. + diff: Print a diff instead of writing. + options: Opt-in transforms to apply. + + Returns: + The :class:`FormatOutcome` for this file. + + """ + if len(entries) > 1: + console.print(f"[yellow] Skipped {location}: multi-document files are not supported by format") + return FormatOutcome.SKIPPED + + schema_file = entries[0] + if not schema_file.valid or schema_file.content is None: + console.print(f"[red] {location}: {schema_file.error_message or 'invalid file'}") + return FormatOutcome.ERROR + + if not is_schema_document(schema_file.content): + return FormatOutcome.SKIPPED + + original = location.read_text(encoding="utf-8") + try: + formatted = format_schema_text(original, options) + except FormatError as exc: + console.print(f"[red] {location}: {exc}") + return FormatOutcome.ERROR + + if formatted == original: + return FormatOutcome.UNCHANGED + + if diff: + _print_schema_diff(location=location, original=original, formatted=formatted) + elif check: + console.print(f"[yellow] Would reformat {location}") + else: + location.write_text(formatted, encoding="utf-8") + console.print(f"[green] Reformatted {location}") + return FormatOutcome.CHANGED + + +@app.command(name="format") +@catch_exception(console=console) +def schema_format( + schemas: list[Path], + check: bool = typer.Option(False, "--check", help="Do not write files; exit 1 if any file would be reformatted."), + diff: bool = typer.Option(False, "--diff", help="Print a diff of the changes instead of writing files."), + strip_defaults: bool = typer.Option( + False, "--strip-defaults", help="Remove attribute/relationship/node keys whose value equals the schema default." + ), + sort_by_order_weight: bool = typer.Option( + False, + "--sort-by-order-weight", + help="Sort attributes and relationships by order_weight (items without one keep their order and go last).", + ), + backfill_order_weight: bool = typer.Option( + False, + "--backfill-order-weight", + help=f"Give attributes/relationships that lack an order_weight the value {DEFAULT_BACKFILL_ORDER_WEIGHT}.", + ), + _: str = CONFIG_PARAM, +) -> None: + """Format Infrahub schema files with a canonical key ordering. + + Reorders the keys within each node, generic, attribute, relationship and + dropdown choice into a consistent, opinionated order so schema files read + the same way and produce small diffs. + + Only your own nodes are formatted; nodes in Infrahub-reserved namespaces are + left untouched. Comments, quoting, and inline (flow) sequences are preserved. + + By default the change is purely key ordering. The opt-in flags additionally + change content: --strip-defaults drops redundant default values, + --sort-by-order-weight reorders attributes/relationships, and + --backfill-order-weight fills in a missing order_weight. + + \b + Examples: + infrahubctl schema format schemas/ + infrahubctl schema format schemas/dcim.yml --diff + infrahubctl schema format schemas/ --check + infrahubctl schema format schemas/ --strip-defaults --sort-by-order-weight + """ + options = FormatOptions( + strip_defaults=strip_defaults, + sort_by_order_weight=sort_by_order_weight, + backfill_order_weight=backfill_order_weight, + ) + schema_files = SchemaFile.load_from_disk(paths=schemas) + + # A genuine multi-document file yields several SchemaFile entries for the + # same location. The per-file ``multiple_documents`` flag is unreliable + # (it is set from a naive `---` substring count that also matches `---` + # inside comments), so group by location and count real documents instead. + entries_by_location: dict[Path, list[SchemaFile]] = {} + for schema_file in schema_files: + entries_by_location.setdefault(schema_file.location, []).append(schema_file) + + reformatted = 0 + unchanged = 0 + would_change = 0 + has_error = False + + for location, entries in entries_by_location.items(): + status = _format_one_schema_file(location=location, entries=entries, check=check, diff=diff, options=options) + if status is FormatOutcome.ERROR: + has_error = True + elif status is FormatOutcome.UNCHANGED: + unchanged += 1 + elif status is FormatOutcome.CHANGED: + if check or diff: + would_change += 1 + else: + reformatted += 1 + + if check or diff: + console.print(f"\n[bold]{would_change} file(s) would be reformatted, {unchanged} unchanged.") + else: + console.print(f"\n[bold]{reformatted} file(s) reformatted, {unchanged} unchanged.") + + if has_error or (check and would_change): + raise typer.Exit(1) diff --git a/infrahub_sdk/ctl/schema_drift.py b/infrahub_sdk/ctl/schema_drift.py new file mode 100644 index 000000000..b57457a83 --- /dev/null +++ b/infrahub_sdk/ctl/schema_drift.py @@ -0,0 +1,105 @@ +"""Detect drift between the Infrahub JSON schema and the formatter's baseline. + +The canonical key ordering in :mod:`infrahub_sdk.ctl.schema_format` is written +against a known set of schema properties. When Infrahub adds, removes, or +renames a property in the published JSON schema +(https://schema.infrahub.app/infrahub/schema/latest.json), that ordering may +need updating so the new key lands in a sensible slot rather than being +preserved as an unrecognised key. + +This module compares the live schema against a committed baseline +(``schema_properties.json``) and reports the difference. It backs the +``schema-drift-check`` invoke task (a warn-only CI step) and the +``schema-drift-update`` task that refreshes the baseline. It never raises on +drift — reporting is the caller's job. +""" + +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any + +import httpx + +from .schema_format import SCHEMA_URL + +# The JSON-schema ``$defs`` whose property sets the formatter orders. Each maps +# to a canonical key list in ``schema_format`` (nodes/generics, attributes, +# relationships, dropdown choices, and node extensions). +TRACKED_DEFINITIONS = [ + "NodeSchema", + "GenericSchema", + "AttributeSchema", + "RelationshipSchema", + "DropdownChoice", + "NodeExtensionSchema", +] + +BASELINE_PATH = Path(__file__).parent / "schema_properties.json" + + +def extract_properties(schema: dict[str, Any]) -> dict[str, list[str]]: + """Extract the sorted property names of each tracked definition. + + Args: + schema: The parsed JSON schema document. + + Returns: + A mapping of definition name to its sorted list of property names. + + """ + definitions = schema.get("$defs") or schema.get("definitions") or {} + return {name: sorted(definitions.get(name, {}).get("properties", {})) for name in TRACKED_DEFINITIONS} + + +def fetch_live_properties(url: str = SCHEMA_URL, timeout: float = 30.0) -> dict[str, list[str]]: + """Fetch the live JSON schema and return its tracked property sets. + + Args: + url: The schema URL to fetch. + timeout: Request timeout in seconds. + + Returns: + A mapping of definition name to its sorted list of property names. + + Raises: + httpx.HTTPError: If the schema cannot be fetched. + + """ + response = httpx.get(url, timeout=timeout, follow_redirects=True) + response.raise_for_status() + return extract_properties(response.json()) + + +def load_baseline(path: Path = BASELINE_PATH) -> dict[str, list[str]]: + """Load the committed baseline property sets.""" + return json.loads(path.read_text(encoding="utf-8")) + + +def write_baseline(properties: dict[str, list[str]], path: Path = BASELINE_PATH) -> None: + """Write ``properties`` to the baseline file as sorted, indented JSON.""" + path.write_text(json.dumps(properties, indent=2, sort_keys=True) + "\n", encoding="utf-8") + + +def compute_drift(live: dict[str, list[str]], baseline: dict[str, list[str]]) -> dict[str, dict[str, list[str]]]: + """Compare live and baseline property sets. + + Args: + live: Property sets from the live schema. + baseline: Property sets from the committed baseline. + + Returns: + A mapping of definition name to ``{"added": [...], "removed": [...]}``, + containing only the definitions that changed. + + """ + drift: dict[str, dict[str, list[str]]] = {} + for name in TRACKED_DEFINITIONS: + live_set = set(live.get(name, [])) + baseline_set = set(baseline.get(name, [])) + added = sorted(live_set - baseline_set) + removed = sorted(baseline_set - live_set) + if added or removed: + drift[name] = {"added": added, "removed": removed} + return drift diff --git a/infrahub_sdk/ctl/schema_format.py b/infrahub_sdk/ctl/schema_format.py new file mode 100644 index 000000000..4decb8754 --- /dev/null +++ b/infrahub_sdk/ctl/schema_format.py @@ -0,0 +1,441 @@ +"""Opinionated formatter for Infrahub schema YAML files. + +The formatter's core responsibility is the *ordering of keys* (lines) within +each node, generic, attribute, relationship and dropdown choice, so that +hand-authored schema files read consistently and produce small diffs. + +By default the transformation is purely cosmetic and semantics-preserving: +only line order changes, and :func:`format_schema_text` re-parses its own +output and raises if the reloaded data differs from the input. + +Three opt-in transforms (see :class:`FormatOptions`) go further and *do* change +what is written — each is off by default and neutralised in the safety check so +only its intended effect is allowed: + +- ``strip_defaults`` — drop keys whose value equals the schema default (context + aware: ``optional: true`` is redundant on a relationship but meaningful on an + attribute). +- ``sort_by_order_weight`` — sort attributes and relationships ascending by + ``order_weight``; items without one keep their authored order and go last. +- ``backfill_order_weight`` — give attributes/relationships that lack an + ``order_weight`` a single constant value. + +Formatting is done with ``ruamel.yaml`` in round-trip mode, so comments (the +``# yaml-language-server`` header, standalone notes, and inline comments), +quoting style, and flow-style sequences (e.g. ``[manufacturer, name__value]``) +are preserved. Standalone comments sitting *between* attributes/relationships +may not follow their item when ``sort_by_order_weight`` reorders the list; +inline comments on a value always travel with it. +""" + +from __future__ import annotations + +import json +import re +from dataclasses import dataclass +from io import StringIO +from typing import Any + +import yaml +from ruamel.yaml import YAML, YAMLError + +# Mirrors ``infrahub.core.constants.RESTRICTED_NAMESPACES``. Kept as a local +# copy because the SDK does not depend on the Infrahub backend. This list is +# stable; if it drifts, a node in a newly restricted namespace would simply be +# formatted like a user node (a harmless outcome for a line-ordering tool). +RESTRICTED_NAMESPACES: list[str] = [ + "Account", + "Branch", + "Builtin", + "Core", + "Deprecated", + "Diff", + "Infrahub", + "Internal", + "Lineage", + "Schema", + "Profile", + "Template", +] + +SCHEMA_URL = "https://schema.infrahub.app/infrahub/schema/latest.json" +SCHEMA_HEADER = f"---\n# yaml-language-server: $schema={SCHEMA_URL}\n" + +# order_weight has no numeric default in the schema (the UI falls back to +# declaration order), so backfill writes this constant. +DEFAULT_BACKFILL_ORDER_WEIGHT = 1000 + +# Matches a real yaml-language-server directive: a comment line whose first +# non-whitespace content is ``# yaml-language-server:``. Deliberately does not +# match the substring appearing in a scalar value or an unrelated comment. +_LANGUAGE_SERVER_HEADER_RE = re.compile(r"^[ \t]*#[ \t]*yaml-language-server[ \t]*:", re.MULTILINE) + +# Canonical key orders. Each pair is (leading keys, trailing keys); any key not +# listed is preserved in its original position between the two groups so the +# formatter never drops data. +FILE_ORDER = ["version", "generics", "nodes", "extensions"] + +NODE_ORDER = [ + "name", + "namespace", + "description", + "label", + "icon", + "documentation", + "include_in_menu", + "menu_placement", + "inherit_from", + "parent", + "children", + "hierarchical", + "default_filter", + "human_friendly_id", + "order_by", + "display_label", + "display_labels", + "uniqueness_constraints", + "generate_profile", + "generate_template", + "used_by", + "restricted_namespaces", + "branch", + "state", +] +NODE_LAST = ["attributes", "relationships"] + +ATTRIBUTE_ORDER = [ + "name", + "kind", + "label", + "unique", + "read_only", + "computed_attribute", + "default_value", + "enum", + "choices", + "regex", + "min_length", + "max_length", + "parameters", + "optional", + "description", + "allow_override", + "branch", + "deprecation", + "state", +] +ATTRIBUTE_LAST = ["order_weight"] + +RELATIONSHIP_ORDER = [ + "name", + "peer", + "label", + "kind", + "cardinality", + "optional", + "identifier", + "direction", + "on_delete", + "hierarchical", + "min_count", + "max_count", + "common_parent", + "common_relatives", + "read_only", + "allow_override", + "branch", + "deprecation", + "state", + "description", +] +RELATIONSHIP_LAST = ["order_weight"] + +CHOICE_ORDER = ["name", "label", "description", "color"] + +EXTENSION_NODE_ORDER = ["kind", "inherit_from"] +EXTENSION_NODE_LAST = ["attributes", "relationships"] + +# Strippable defaults, grounded in the published JSON schema's ``default`` +# values. Consequential or internal fields (``branch``, ``state``, +# ``inherited``, ``display``) are intentionally excluded: stripping an explicit +# value there would couple the schema to whatever the default happens to be at +# load time. +ENTITY_DEFAULTS: dict[str, Any] = { + "generate_profile": True, + "generate_template": False, + "hierarchical": False, +} +ATTRIBUTE_DEFAULTS: dict[str, Any] = { + "read_only": False, + "unique": False, + "optional": False, + "allow_override": "any", +} +RELATIONSHIP_DEFAULTS: dict[str, Any] = { + "kind": "Generic", + "cardinality": "many", + "optional": True, + "direction": "bidirectional", + "read_only": False, + "allow_override": "any", + "min_count": 0, + "max_count": 0, +} + + +@dataclass(frozen=True) +class FormatOptions: + """Opt-in transforms that change file content beyond key ordering.""" + + strip_defaults: bool = False + sort_by_order_weight: bool = False + backfill_order_weight: bool = False + + +class FormatError(Exception): + """Raised when formatting would change the meaning of a schema file.""" + + +def _build_yaml() -> YAML: + """Return a round-trip YAML handler configured to match the schema-library style.""" + yaml_handler = YAML() + yaml_handler.preserve_quotes = True + # Schema files begin with a `---` document-start marker; keep it. + yaml_handler.explicit_start = True + # Match the schema-library layout: block sequences indented under their key + # (`attributes:\n - name: ...`). + yaml_handler.indent(mapping=2, sequence=4, offset=2) + # A very wide value keeps long scalars (descriptions, Jinja2 templates) on + # their original line instead of being re-wrapped. + yaml_handler.width = 4096 + return yaml_handler + + +def reorder_mapping(mapping: Any, leading: list[str], trailing: list[str]) -> None: + """Reorder a mapping's keys in place into canonical order. + + Keys in ``leading`` come first (in that order), keys in ``trailing`` come + last (in that order), and any remaining keys keep their original relative + order in between. Reordering is done in place with ``move_to_end`` so the + comments ruamel attaches to each key travel with it. + + Args: + mapping: The (round-trip) mapping to reorder. + leading: Keys to place first, in order. + trailing: Keys to force to the end, in order. + + """ + # Round-trip maps (and OrderedDict) support move_to_end; anything else + # (a scalar, a plain list) is left as-is. + if not hasattr(mapping, "move_to_end"): + return + + known = set(leading) | set(trailing) + ordered_keys = [key for key in leading if key in mapping] + ordered_keys += [key for key in mapping if key not in known] + ordered_keys += [key for key in trailing if key in mapping] + + for key in ordered_keys: + mapping.move_to_end(key) + + +def _strip_default_keys(mapping: dict[str, Any], defaults: dict[str, Any]) -> None: + """Remove keys whose value equals the schema default.""" + for key, default in defaults.items(): + if key in mapping and mapping[key] == default: + del mapping[key] + + +def _order_weight_sort_key(item: Any) -> float: + weight = item.get("order_weight") if isinstance(item, dict) else None + # Missing/non-numeric weights sort last; a stable sort keeps their order. + return weight if isinstance(weight, int) and not isinstance(weight, bool) else float("inf") + + +def _format_item(item: Any, defaults: dict[str, Any], leading: list[str], options: FormatOptions) -> None: + if not isinstance(item, dict): + return + if options.backfill_order_weight and "order_weight" not in item: + item["order_weight"] = DEFAULT_BACKFILL_ORDER_WEIGHT + if options.strip_defaults: + _strip_default_keys(item, defaults) + reorder_mapping(item, leading, ["order_weight"]) + if defaults is ATTRIBUTE_DEFAULTS: + choices = item.get("choices") + if isinstance(choices, list): + for choice in choices: + reorder_mapping(choice, CHOICE_ORDER, []) + + +def _format_item_list(items: Any, defaults: dict[str, Any], leading: list[str], options: FormatOptions) -> None: + if not isinstance(items, list): + return + for item in items: + _format_item(item, defaults, leading, options) + if options.sort_by_order_weight: + items.sort(key=_order_weight_sort_key) + + +def _format_entity(entity: Any, leading: list[str], trailing: list[str], options: FormatOptions) -> None: + """Reorder an entity's own keys, then transform its attributes and relationships.""" + if options.strip_defaults: + _strip_default_keys(entity, ENTITY_DEFAULTS) + reorder_mapping(entity, leading, trailing) + _format_item_list(entity.get("attributes"), ATTRIBUTE_DEFAULTS, ATTRIBUTE_ORDER, options) + _format_item_list(entity.get("relationships"), RELATIONSHIP_DEFAULTS, RELATIONSHIP_ORDER, options) + + +def _is_restricted(entity: Any) -> bool: + return isinstance(entity, dict) and entity.get("namespace") in RESTRICTED_NAMESPACES + + +def format_document(data: Any, options: FormatOptions | None = None) -> None: + """Reorder (and optionally transform) a parsed schema document in place. + + Nodes and generics in a restricted namespace are left untouched. Extension + entries are always formatted, since the extension block itself is authored + by the user regardless of which node it extends. + + Args: + data: The parsed (round-trip) schema document. + options: Opt-in transforms; defaults to key-ordering only. + + """ + options = options or FormatOptions() + reorder_mapping(data, FILE_ORDER, []) + + for section in ("generics", "nodes"): + entities = data.get(section) + if not isinstance(entities, list): + continue + for entity in entities: + if isinstance(entity, dict) and not _is_restricted(entity): + _format_entity(entity, NODE_ORDER, NODE_LAST, options) + + extensions = data.get("extensions") + if isinstance(extensions, dict) and isinstance(extensions.get("nodes"), list): + for entity in extensions["nodes"]: + if isinstance(entity, dict): + _format_entity(entity, EXTENSION_NODE_ORDER, EXTENSION_NODE_LAST, options) + + +def _ensure_schema_header(text: str) -> str: + """Add the canonical ``# yaml-language-server`` header if the file lacks one. + + Only an actual header *directive line* counts as present — a bare + ``yaml-language-server`` substring elsewhere (in a scalar value or an + unrelated comment) must not suppress the header. + """ + if _LANGUAGE_SERVER_HEADER_RE.search(text): + return text + if text.startswith("---\n"): + return SCHEMA_HEADER + text[len("---\n") :] + return SCHEMA_HEADER + text + + +def is_schema_document(content: Any) -> bool: + """Return True if ``content`` looks like an Infrahub schema file.""" + return ( + isinstance(content, dict) + and "version" in content + and any(key in content for key in ("nodes", "generics", "extensions")) + ) + + +def _normalize_item(item: Any, defaults: dict[str, Any], options: FormatOptions) -> Any: + if not isinstance(item, dict): + return item + normalized = dict(item) + if options.strip_defaults: + for key, default in defaults.items(): + normalized.setdefault(key, default) + if options.backfill_order_weight: + normalized.setdefault("order_weight", DEFAULT_BACKFILL_ORDER_WEIGHT) + return normalized + + +def _normalize_entity(entity: Any, options: FormatOptions) -> Any: + if not isinstance(entity, dict): + return entity + normalized = dict(entity) + if options.strip_defaults: + for key, default in ENTITY_DEFAULTS.items(): + normalized.setdefault(key, default) + for key, defaults in (("attributes", ATTRIBUTE_DEFAULTS), ("relationships", RELATIONSHIP_DEFAULTS)): + items = normalized.get(key) + if isinstance(items, list): + items = [_normalize_item(item, defaults, options) for item in items] + if options.sort_by_order_weight: + # Sort by full item content, not by name or weight (both of + # which can repeat): a total, content-based order lets the guard + # permit any reorder while still catching a dropped or corrupted + # item. + items = sorted(items, key=lambda it: json.dumps(it, sort_keys=True, default=str)) + normalized[key] = items + return normalized + + +def _normalize_for_guard(data: Any, options: FormatOptions) -> Any: + """Collapse exactly the intended transforms so the guard permits them. + + The same normalisation is applied to the input and the formatted output, so + an intended change (a stripped default, a reordered list, a backfilled + weight) is neutralised on both sides while any *unintended* corruption still + causes inequality. With no options set this is effectively an identity. + """ + if not isinstance(data, dict): + return data + normalized = dict(data) + for section in ("generics", "nodes"): + entities = normalized.get(section) + if isinstance(entities, list): + normalized[section] = [_normalize_entity(entity, options) for entity in entities] + extensions = normalized.get("extensions") + if isinstance(extensions, dict) and isinstance(extensions.get("nodes"), list): + extensions = dict(extensions) + extensions["nodes"] = [_normalize_entity(entity, options) for entity in extensions["nodes"]] + normalized["extensions"] = extensions + return normalized + + +def format_schema_text(raw_text: str, options: FormatOptions | None = None) -> str: + """Format the text of a schema file into canonical YAML text. + + Args: + raw_text: The original file contents. + options: Opt-in transforms; defaults to key-ordering only. + + Returns: + The formatted YAML text, with comments and quoting preserved. + + Raises: + FormatError: If the file cannot be parsed as round-trip YAML (e.g. a + duplicate key), or if formatting would change the file's meaning + beyond the transforms requested via ``options``. + + """ + options = options or FormatOptions() + yaml_handler = _build_yaml() + try: + # Round-trip loading is stricter than the PyYAML safe_load used to + # discover schema files (e.g. it rejects duplicate keys). Convert that + # into a per-file FormatError so one bad file does not abort the run. + data = yaml_handler.load(raw_text) + except YAMLError as exc: + raise FormatError(f"could not parse as YAML: {exc}") from exc + + if not is_schema_document(data): + return raw_text + + format_document(data, options) + + buffer = StringIO() + yaml_handler.dump(data, buffer) + text = _ensure_schema_header(buffer.getvalue()) + + original = _normalize_for_guard(yaml.safe_load(raw_text), options) + formatted = _normalize_for_guard(yaml.safe_load(text), options) + if original != formatted: + raise FormatError("Formatting would change the schema content; aborting to avoid data loss.") + + return text diff --git a/infrahub_sdk/ctl/schema_properties.json b/infrahub_sdk/ctl/schema_properties.json new file mode 100644 index 000000000..ae2876d82 --- /dev/null +++ b/infrahub_sdk/ctl/schema_properties.json @@ -0,0 +1,120 @@ +{ + "AttributeSchema": [ + "allow_override", + "branch", + "choices", + "computed_attribute", + "default_value", + "deprecation", + "description", + "display", + "enum", + "id", + "inherited", + "kind", + "label", + "max_length", + "min_length", + "name", + "optional", + "order_weight", + "parameters", + "read_only", + "regex", + "state", + "unique" + ], + "DropdownChoice": [ + "color", + "description", + "id", + "label", + "name", + "state" + ], + "GenericSchema": [ + "attributes", + "branch", + "default_filter", + "description", + "display_label", + "display_labels", + "documentation", + "generate_profile", + "hierarchical", + "human_friendly_id", + "icon", + "id", + "include_in_menu", + "label", + "menu_placement", + "name", + "namespace", + "order_by", + "relationships", + "restricted_namespaces", + "state", + "uniqueness_constraints", + "used_by" + ], + "NodeExtensionSchema": [ + "attributes", + "id", + "kind", + "relationships", + "state" + ], + "NodeSchema": [ + "attributes", + "branch", + "children", + "default_filter", + "description", + "display_label", + "display_labels", + "documentation", + "generate_profile", + "generate_template", + "hierarchy", + "human_friendly_id", + "icon", + "id", + "include_in_menu", + "inherit_from", + "label", + "menu_placement", + "name", + "namespace", + "order_by", + "parent", + "relationships", + "state", + "uniqueness_constraints" + ], + "RelationshipSchema": [ + "allow_override", + "branch", + "cardinality", + "common_parent", + "common_relatives", + "deprecation", + "description", + "direction", + "display", + "hierarchical", + "id", + "identifier", + "inherited", + "kind", + "label", + "max_count", + "min_count", + "name", + "on_delete", + "optional", + "order_weight", + "peer", + "read_only", + "state" + ] +} diff --git a/infrahub_sdk/ctl/validate.py b/infrahub_sdk/ctl/validate.py index cf69e2fa5..8b03b332e 100644 --- a/infrahub_sdk/ctl/validate.py +++ b/infrahub_sdk/ctl/validate.py @@ -5,14 +5,15 @@ import typer import ujson -from pydantic import ValidationError from rich.console import Console +from rich.markup import escape from ..async_typer import AsyncTyper -from ..ctl.client import initialize_client, initialize_client_sync +from ..ctl.client import initialize_client_sync from ..ctl.exceptions import QueryNotFoundError from ..ctl.utils import catch_exception, find_graphql_query, parse_cli_vars from ..exceptions import GraphQLError +from ..schema import validate_schema as validate_schema_offline from ..utils import write_to_file from ..yaml import SchemaFile from .parameters import CONFIG_PARAM @@ -36,16 +37,16 @@ async def validate_schema(schema: Path, _: str = CONFIG_PARAM) -> None: console.print(f"[red]Unable to find {schema}") raise typer.Exit(1) - client = initialize_client() + result = validate_schema_offline(schema=schema_data[0].payload) - try: - client.schema.validate(schema_data[0].payload) - except ValidationError as exc: - console.print(f"[red]Schema not valid, found {len(exc.errors())} error(s)") - for error in exc.errors(): - loc_str = [str(item) for item in error["loc"]] - console.print(f" '{'/'.join(loc_str)}' | {error['msg']} ({error['type']})") - raise typer.Exit(1) from None + for warning in result.warnings: + console.print(f"[yellow]{escape(warning.message)}") + + if not result.valid: + console.print(f"[red]Schema not valid, found {len(result.errors)} error(s)") + for error in result.errors: + console.print(f" {escape(error.message)}") + raise typer.Exit(1) console.print("[green]Schema is valid !!") diff --git a/infrahub_sdk/data.py b/infrahub_sdk/data.py index 1539cce9e..6a28487a9 100644 --- a/infrahub_sdk/data.py +++ b/infrahub_sdk/data.py @@ -5,6 +5,11 @@ from .node import InfrahubNode # noqa: TC001 +class ServerInfo(BaseModel): + version: str = "" + deployment_id: str = "" + + class RepositoryBranchInfo(BaseModel): internal_status: str diff --git a/infrahub_sdk/exceptions.py b/infrahub_sdk/exceptions.py index f0774c2dd..02111b9ac 100644 --- a/infrahub_sdk/exceptions.py +++ b/infrahub_sdk/exceptions.py @@ -22,6 +22,24 @@ def __init__(self, message: str | None = None, content: str | None = None, url: super().__init__(self.message) +class RateLimitError(Error): + """Raised when a request keeps receiving HTTP 429 past the configured retry budget.""" + + def __init__( + self, + url: str, + attempts: int, + retry_after: float | None = None, + message: str | None = None, + ) -> None: + self.url = url + self.attempts = attempts + self.retry_after = retry_after + if message is None: + message = f"Request to {url} was rate-limited (HTTP 429) after {attempts} attempt(s)." + super().__init__(message) + + class ServerNotReachableError(Error): def __init__(self, address: str, message: str | None = None) -> None: self.address = address diff --git a/infrahub_sdk/node/__init__.py b/infrahub_sdk/node/__init__.py index 6bbde8540..eb55bfe36 100644 --- a/infrahub_sdk/node/__init__.py +++ b/infrahub_sdk/node/__init__.py @@ -6,6 +6,7 @@ ARTIFACT_FETCH_FEATURE_NOT_SUPPORTED_MESSAGE, ARTIFACT_GENERATE_FEATURE_NOT_SUPPORTED_MESSAGE, HFID_STR_SEPARATOR, + IP_ADDRESS_TYPES, IP_TYPES, MATCHES_LOCAL_CHECKSUM_FEATURE_NOT_SUPPORTED_MESSAGE, PROPERTIES_FLAG, @@ -30,6 +31,7 @@ "ARTIFACT_FETCH_FEATURE_NOT_SUPPORTED_MESSAGE", "ARTIFACT_GENERATE_FEATURE_NOT_SUPPORTED_MESSAGE", "HFID_STR_SEPARATOR", + "IP_ADDRESS_TYPES", "IP_TYPES", "MATCHES_LOCAL_CHECKSUM_FEATURE_NOT_SUPPORTED_MESSAGE", "PROPERTIES_FLAG", diff --git a/infrahub_sdk/node/attribute.py b/infrahub_sdk/node/attribute.py index d70d7ee93..4ada6b9de 100644 --- a/infrahub_sdk/node/attribute.py +++ b/infrahub_sdk/node/attribute.py @@ -5,7 +5,14 @@ from typing import TYPE_CHECKING, Any, NamedTuple, get_args from ..uuidt import UUIDT -from .constants import ATTRIBUTE_METADATA_OBJECT, IP_TYPES, PROPERTIES_FLAG, PROPERTIES_OBJECT, SAFE_VALUE +from .constants import ( + ATTRIBUTE_METADATA_OBJECT, + IP_ADDRESS_TYPES, + IP_TYPES, + PROPERTIES_FLAG, + PROPERTIES_OBJECT, + SAFE_VALUE, +) from .property import NodeProperty if TYPE_CHECKING: @@ -67,8 +74,8 @@ class Attribute: def __init__(self, name: str, schema: AttributeSchemaAPI, data: Any | dict) -> None: """Build an ``Attribute`` from raw GraphQL data. - IP-typed attributes (``IPHost``, ``IPNetwork``) are parsed via the standard - ``ipaddress`` module so the in-memory value is a network/interface object. + IP-typed attributes (``IPHost``, ``IPNetwork``, ``IPAddress``) are parsed via the standard + ``ipaddress`` module so the in-memory value is an interface, network or address object. Args: name (str): The name of the attribute. @@ -105,6 +112,7 @@ def __init__(self, name: str, schema: AttributeSchemaAPI, data: Any | dict) -> N value_mapper: dict[str, Callable] = { "IPHost": ipaddress.ip_interface, "IPNetwork": ipaddress.ip_network, + "IPAddress": ipaddress.ip_address, } mapper = value_mapper.get(schema.kind, lambda value: value) self._value = mapper(data.get("value")) @@ -160,7 +168,13 @@ def _initialize_graphql_payload(self) -> _GraphQLPayloadAttribute: ) # Safe strings, IP types, and everything else - value = self.value.with_prefixlen if isinstance(self.value, get_args(IP_TYPES)) else self.value + if isinstance(self.value, get_args(IP_TYPES)): + value = self.value.with_prefixlen + elif isinstance(self.value, get_args(IP_ADDRESS_TYPES)): + # bare addresses have no prefix; serialize their canonical string form + value = str(self.value) + else: + value = self.value return _GraphQLPayloadAttribute(payload={"value": value}, variables={}, needs_metadata=True) def _generate_input_data(self) -> _GraphQLPayloadAttribute: diff --git a/infrahub_sdk/node/constants.py b/infrahub_sdk/node/constants.py index 6a56584ed..192bbf6de 100644 --- a/infrahub_sdk/node/constants.py +++ b/infrahub_sdk/node/constants.py @@ -18,6 +18,9 @@ IP_TYPES = ipaddress.IPv4Interface | ipaddress.IPv6Interface | ipaddress.IPv4Network | ipaddress.IPv6Network +# Bare IP addresses (no prefix); serialized with str() rather than with_prefixlen +IP_ADDRESS_TYPES = ipaddress.IPv4Address | ipaddress.IPv6Address + ARTIFACT_FETCH_FEATURE_NOT_SUPPORTED_MESSAGE = ( "calling artifact_fetch is only supported for nodes that are Artifact Definition target" ) diff --git a/infrahub_sdk/node/node.py b/infrahub_sdk/node/node.py index b23f6f865..df1ec65b5 100644 --- a/infrahub_sdk/node/node.py +++ b/infrahub_sdk/node/node.py @@ -6,7 +6,7 @@ from pathlib import Path from typing import TYPE_CHECKING, Any, BinaryIO, overload -from ..constants import InfrahubClientMode +from ..constants import InfrahubClientMode, Priority from ..exceptions import ( FeatureNotSupportedError, NodeNotFoundError, @@ -282,14 +282,15 @@ def __setattr__(self, name: str, value: Any) -> None: super().__setattr__(name, value) def _get_request_context(self, request_context: RequestContext | None = None) -> dict[str, Any] | None: + # priority rides the X-Priority header, not the mutation body — the server context input has no such field if request_context: - return request_context.model_dump(exclude_none=True) + return request_context.model_dump(exclude_none=True, exclude={"priority"}) or None client: InfrahubClient | InfrahubClientSync | None = getattr(self, "_client", None) if not client or not client.request_context: return None - return client.request_context.model_dump(exclude_none=True) + return client.request_context.model_dump(exclude_none=True, exclude={"priority"}) or None def _init_relationships(self, data: dict | None = None) -> None: pass @@ -651,8 +652,8 @@ def _validate_file_object_support(self, message: str) -> None: def generate_query_data_init( self, filters: dict[str, Any] | None = None, - offset: int | None = None, - limit: int | None = None, + offset: int | str | None = None, + limit: int | str | None = None, include: list[str] | None = None, exclude: list[str] | None = None, partial_match: bool = False, @@ -667,8 +668,10 @@ def generate_query_data_init( Args: filters (dict[str, Any], optional): Filters to apply to the query. - offset (int, optional): Pagination offset. - limit (int, optional): Pagination limit. + offset (int | str, optional): Pagination offset, either a literal value or a + GraphQL variable placeholder such as ``"$offset"``. + limit (int | str, optional): Pagination limit, either a literal value or a + GraphQL variable placeholder such as ``"$limit"``. include (list[str], optional): Attributes or relationships to include. exclude (list[str], optional): Attributes or relationships to exclude. partial_match (bool, optional): When ``True``, allow partial matches on filter @@ -1211,7 +1214,12 @@ async def upload_if_changed( return UploadResult(was_uploaded=True, checksum=local_digest) - async def delete(self, timeout: int | None = None, request_context: RequestContext | None = None) -> None: + async def delete( + self, + timeout: int | None = None, + request_context: RequestContext | None = None, + priority: Priority | None = None, + ) -> None: """Delete this node on the backend. Args: @@ -1219,6 +1227,8 @@ async def delete(self, timeout: int | None = None, request_context: RequestConte GraphQL API. Specified in seconds. request_context (RequestContext, optional): Request-level context passed through to the mutation. When omitted, the client's request context is used. + priority (Priority, optional): Per-request priority emitted as the X-Priority header, + overriding the client default for this request only. """ input_data = {"data": {"id": self.id}} @@ -1236,6 +1246,7 @@ async def delete(self, timeout: int | None = None, request_context: RequestConte branch_name=self._branch, timeout=timeout, tracker=f"mutation-{str(self._schema.kind).lower()}-delete", + priority=priority, ) async def save( @@ -1244,6 +1255,7 @@ async def save( update_group_context: bool | None = None, timeout: int | None = None, request_context: RequestContext | None = None, + priority: Priority | None = None, ) -> None: """Persist this node to the backend, creating or updating it as appropriate. @@ -1262,12 +1274,16 @@ async def save( GraphQL API. Specified in seconds. request_context (RequestContext, optional): Request-level context passed through to the mutation. When omitted, the client's request context is used. + priority (Priority, optional): Per-request priority emitted as the X-Priority header, + overriding the client default for this request only. """ if self._existing is False or allow_upsert is True: - await self.create(allow_upsert=allow_upsert, timeout=timeout, request_context=request_context) + await self.create( + allow_upsert=allow_upsert, timeout=timeout, request_context=request_context, priority=priority + ) else: - await self.update(timeout=timeout, request_context=request_context) + await self.update(timeout=timeout, request_context=request_context, priority=priority) if update_group_context is None and self._client.mode == InfrahubClientMode.TRACKING: update_group_context = True @@ -1340,8 +1356,8 @@ async def _process_hierarchical_fields( async def generate_query_data( self, filters: dict[str, Any] | None = None, - offset: int | None = None, - limit: int | None = None, + offset: int | str | None = None, + limit: int | str | None = None, include: list[str] | None = None, exclude: list[str] | None = None, fragment: bool = False, @@ -1360,8 +1376,10 @@ async def generate_query_data( Args: filters (dict[str, Any], optional): Filters to apply to the query. - offset (int, optional): Pagination offset. - limit (int, optional): Pagination limit. + offset (int | str, optional): Pagination offset, either a literal value or a + GraphQL variable placeholder such as ``"$offset"``. + limit (int | str, optional): Pagination limit, either a literal value or a + GraphQL variable placeholder such as ``"$limit"``. include (list[str], optional): Attributes or relationships to include. exclude (list[str], optional): Attributes or relationships to exclude. fragment (bool, optional): When ``True`` and the schema is a generic, emit @@ -1572,7 +1590,11 @@ def _generate_mutation_query(self) -> dict[str, Any]: return query_result async def _process_mutation_result( - self, mutation_name: str, response: dict[str, Any], timeout: int | None = None + self, + mutation_name: str, + response: dict[str, Any], + timeout: int | None = None, + priority: Priority | None = None, ) -> None: object_response: dict[str, Any] = response[mutation_name]["object"] self.id = object_response["id"] @@ -1596,11 +1618,15 @@ async def _process_mutation_result( related_node = RelatedNode( client=self._client, branch=self._branch, schema=rel.schema, data=allocated_resource ) - await related_node.fetch(timeout=timeout) + await related_node.fetch(timeout=timeout, priority=priority) setattr(self, rel_name, related_node) async def create( - self, allow_upsert: bool = False, timeout: int | None = None, request_context: RequestContext | None = None + self, + allow_upsert: bool = False, + timeout: int | None = None, + request_context: RequestContext | None = None, + priority: Priority | None = None, ) -> None: """Create this node on the backend. @@ -1618,6 +1644,8 @@ async def create( GraphQL API. Specified in seconds. request_context (RequestContext, optional): Request-level context passed through to the mutation. When omitted, the client's request context is used. + priority (Priority, optional): Per-request priority emitted as the X-Priority header, + overriding the client default for this request only. Raises: ValueError: If this is a file-object node and no file content has been set. @@ -1662,6 +1690,7 @@ async def create( branch_name=self._branch, tracker=tracker, timeout=timeout, + priority=priority, ) finally: if prepared.should_close and prepared.file_object: @@ -1675,11 +1704,18 @@ async def create( tracker=tracker, variables=input_data["variables"], timeout=timeout, + priority=priority, ) - await self._process_mutation_result(mutation_name=mutation_name, response=response, timeout=timeout) + await self._process_mutation_result( + mutation_name=mutation_name, response=response, timeout=timeout, priority=priority + ) async def update( - self, do_full_update: bool = False, timeout: int | None = None, request_context: RequestContext | None = None + self, + do_full_update: bool = False, + timeout: int | None = None, + request_context: RequestContext | None = None, + priority: Priority | None = None, ) -> None: """Update this node on the backend. @@ -1697,6 +1733,8 @@ async def update( GraphQL API. Specified in seconds. request_context (RequestContext, optional): Request-level context passed through to the mutation. When omitted, the client's request context is used. + priority (Priority, optional): Per-request priority emitted as the X-Priority header, + overriding the client default for this request only. """ input_data = self._generate_input_data(exclude_unmodified=not do_full_update, request_context=request_context) @@ -1722,6 +1760,7 @@ async def update( branch_name=self._branch, tracker=tracker, timeout=timeout, + priority=priority, ) finally: if prepared.should_close and prepared.file_object: @@ -1735,8 +1774,11 @@ async def update( timeout=timeout, tracker=tracker, variables=input_data["variables"], + priority=priority, ) - await self._process_mutation_result(mutation_name=mutation_name, response=response, timeout=timeout) + await self._process_mutation_result( + mutation_name=mutation_name, response=response, timeout=timeout, priority=priority + ) async def _process_relationships( self, @@ -1813,30 +1855,37 @@ async def get_pool_allocated_resources(self, resource: InfrahubNode) -> list[Inf graphql_query_name = "InfrahubResourcePoolAllocated" node_ids_per_kind: dict[str, list[str]] = {} + query = Query( + query={ + graphql_query_name: { + "@filters": { + "pool_id": "$pool_id", + "resource_id": "$resource_id", + "offset": "$offset", + "limit": "$limit", + }, + "count": None, + "edges": {"node": {"id": None, "kind": None, "branch": None, "identifier": None}}, + } + }, + name="GetAllocatedResourceForPool", + variables={"pool_id": str, "resource_id": str, "offset": int, "limit": int}, + ) + query_str = query.render() + has_remaining_items = True page_number = 1 while has_remaining_items: page_offset = (page_number - 1) * self._client.pagination_size - query = Query( - query={ - graphql_query_name: { - "@filters": { - "pool_id": "$pool_id", - "resource_id": "$resource_id", - "offset": page_offset, - "limit": self._client.pagination_size, - }, - "count": None, - "edges": {"node": {"id": None, "kind": None, "branch": None, "identifier": None}}, - } - }, - name="GetAllocatedResourceForPool", - variables={"pool_id": str, "resource_id": str}, - ) response = await self._client.execute_graphql( - query=query.render(), - variables={"pool_id": self.id, "resource_id": resource.id}, + query=query_str, + variables={ + "pool_id": self.id, + "resource_id": resource.id, + "offset": page_offset, + "limit": self._client.pagination_size, + }, branch_name=self._branch, tracker=f"get-allocated-resources-page{page_number}", ) @@ -2399,7 +2448,12 @@ def upload_if_changed( return UploadResult(was_uploaded=True, checksum=local_digest) - def delete(self, timeout: int | None = None, request_context: RequestContext | None = None) -> None: + def delete( + self, + timeout: int | None = None, + request_context: RequestContext | None = None, + priority: Priority | None = None, + ) -> None: """Delete this node on the backend. Args: @@ -2407,6 +2461,8 @@ def delete(self, timeout: int | None = None, request_context: RequestContext | N GraphQL API. Specified in seconds. request_context (RequestContext, optional): Request-level context passed through to the mutation. When omitted, the client's request context is used. + priority (Priority, optional): Per-request priority emitted as the X-Priority header, + overriding the client default for this request only. """ input_data = {"data": {"id": self.id}} @@ -2424,6 +2480,7 @@ def delete(self, timeout: int | None = None, request_context: RequestContext | N branch_name=self._branch, tracker=f"mutation-{str(self._schema.kind).lower()}-delete", timeout=timeout, + priority=priority, ) def save( @@ -2432,6 +2489,7 @@ def save( update_group_context: bool | None = None, timeout: int | None = None, request_context: RequestContext | None = None, + priority: Priority | None = None, ) -> None: """Persist this node to the backend, creating or updating it as appropriate. @@ -2450,12 +2508,14 @@ def save( GraphQL API. Specified in seconds. request_context (RequestContext, optional): Request-level context passed through to the mutation. When omitted, the client's request context is used. + priority (Priority, optional): Per-request priority emitted as the X-Priority header, + overriding the client default for this request only. """ if self._existing is False or allow_upsert is True: - self.create(allow_upsert=allow_upsert, timeout=timeout, request_context=request_context) + self.create(allow_upsert=allow_upsert, timeout=timeout, request_context=request_context, priority=priority) else: - self.update(timeout=timeout, request_context=request_context) + self.update(timeout=timeout, request_context=request_context, priority=priority) if update_group_context is None and self._client.mode == InfrahubClientMode.TRACKING: update_group_context = True @@ -2524,8 +2584,8 @@ def _process_hierarchical_fields( def generate_query_data( self, filters: dict[str, Any] | None = None, - offset: int | None = None, - limit: int | None = None, + offset: int | str | None = None, + limit: int | str | None = None, include: list[str] | None = None, exclude: list[str] | None = None, fragment: bool = False, @@ -2544,8 +2604,10 @@ def generate_query_data( Args: filters (dict[str, Any], optional): Filters to apply to the query. - offset (int, optional): Pagination offset. - limit (int, optional): Pagination limit. + offset (int | str, optional): Pagination offset, either a literal value or a + GraphQL variable placeholder such as ``"$offset"``. + limit (int | str, optional): Pagination limit, either a literal value or a + GraphQL variable placeholder such as ``"$limit"``. include (list[str], optional): Attributes or relationships to include. exclude (list[str], optional): Attributes or relationships to exclude. fragment (bool, optional): When ``True`` and the schema is a generic, emit @@ -2759,7 +2821,11 @@ def _generate_mutation_query(self) -> dict[str, Any]: return query_result def _process_mutation_result( - self, mutation_name: str, response: dict[str, Any], timeout: int | None = None + self, + mutation_name: str, + response: dict[str, Any], + timeout: int | None = None, + priority: Priority | None = None, ) -> None: object_response: dict[str, Any] = response[mutation_name]["object"] self.id = object_response["id"] @@ -2783,11 +2849,15 @@ def _process_mutation_result( related_node = RelatedNodeSync( client=self._client, branch=self._branch, schema=rel.schema, data=allocated_resource ) - related_node.fetch(timeout=timeout) + related_node.fetch(timeout=timeout, priority=priority) setattr(self, rel_name, related_node) def create( - self, allow_upsert: bool = False, timeout: int | None = None, request_context: RequestContext | None = None + self, + allow_upsert: bool = False, + timeout: int | None = None, + request_context: RequestContext | None = None, + priority: Priority | None = None, ) -> None: """Create this node on the backend. @@ -2805,6 +2875,8 @@ def create( GraphQL API. Specified in seconds. request_context (RequestContext, optional): Request-level context passed through to the mutation. When omitted, the client's request context is used. + priority (Priority, optional): Per-request priority emitted as the X-Priority header, + overriding the client default for this request only. Raises: ValueError: If this is a file-object node and no file content has been set. @@ -2849,6 +2921,7 @@ def create( branch_name=self._branch, tracker=tracker, timeout=timeout, + priority=priority, ) finally: if prepared.should_close and prepared.file_object: @@ -2862,11 +2935,18 @@ def create( tracker=tracker, variables=input_data["variables"], timeout=timeout, + priority=priority, ) - self._process_mutation_result(mutation_name=mutation_name, response=response, timeout=timeout) + self._process_mutation_result( + mutation_name=mutation_name, response=response, timeout=timeout, priority=priority + ) def update( - self, do_full_update: bool = False, timeout: int | None = None, request_context: RequestContext | None = None + self, + do_full_update: bool = False, + timeout: int | None = None, + request_context: RequestContext | None = None, + priority: Priority | None = None, ) -> None: """Update this node on the backend. @@ -2884,6 +2964,8 @@ def update( GraphQL API. Specified in seconds. request_context (RequestContext, optional): Request-level context passed through to the mutation. When omitted, the client's request context is used. + priority (Priority, optional): Per-request priority emitted as the X-Priority header, + overriding the client default for this request only. """ input_data = self._generate_input_data(exclude_unmodified=not do_full_update, request_context=request_context) @@ -2909,6 +2991,7 @@ def update( branch_name=self._branch, tracker=tracker, timeout=timeout, + priority=priority, ) finally: if prepared.should_close and prepared.file_object: @@ -2922,8 +3005,11 @@ def update( tracker=tracker, variables=input_data["variables"], timeout=timeout, + priority=priority, ) - self._process_mutation_result(mutation_name=mutation_name, response=response, timeout=timeout) + self._process_mutation_result( + mutation_name=mutation_name, response=response, timeout=timeout, priority=priority + ) def _process_relationships( self, @@ -3000,30 +3086,37 @@ def get_pool_allocated_resources(self, resource: InfrahubNodeSync) -> list[Infra graphql_query_name = "InfrahubResourcePoolAllocated" node_ids_per_kind: dict[str, list[str]] = {} + query = Query( + query={ + graphql_query_name: { + "@filters": { + "pool_id": "$pool_id", + "resource_id": "$resource_id", + "offset": "$offset", + "limit": "$limit", + }, + "count": None, + "edges": {"node": {"id": None, "kind": None, "branch": None, "identifier": None}}, + } + }, + name="GetAllocatedResourceForPool", + variables={"pool_id": str, "resource_id": str, "offset": int, "limit": int}, + ) + query_str = query.render() + has_remaining_items = True page_number = 1 while has_remaining_items: page_offset = (page_number - 1) * self._client.pagination_size - query = Query( - query={ - graphql_query_name: { - "@filters": { - "pool_id": "$pool_id", - "resource_id": "$resource_id", - "offset": page_offset, - "limit": self._client.pagination_size, - }, - "count": None, - "edges": {"node": {"id": None, "kind": None, "branch": None, "identifier": None}}, - } - }, - name="GetAllocatedResourceForPool", - variables={"pool_id": str, "resource_id": str}, - ) response = self._client.execute_graphql( - query=query.render(), - variables={"pool_id": self.id, "resource_id": resource.id}, + query=query_str, + variables={ + "pool_id": self.id, + "resource_id": resource.id, + "offset": page_offset, + "limit": self._client.pagination_size, + }, branch_name=self._branch, tracker=f"get-allocated-resources-page{page_number}", ) diff --git a/infrahub_sdk/node/related_node.py b/infrahub_sdk/node/related_node.py index 229aad61f..f36523f13 100644 --- a/infrahub_sdk/node/related_node.py +++ b/infrahub_sdk/node/related_node.py @@ -12,6 +12,7 @@ if TYPE_CHECKING: from ..client import InfrahubClient, InfrahubClientSync + from ..constants import Priority from ..schema import RelationshipSchemaAPI from .node import InfrahubNode, InfrahubNodeBase, InfrahubNodeSync @@ -347,7 +348,7 @@ def __init__( self._client = client super().__init__(branch=branch, schema=schema, data=data, name=name) - async def fetch(self, timeout: int | None = None) -> None: + async def fetch(self, timeout: int | None = None, priority: Priority | None = None) -> None: """Fetch the full peer node from the backend and cache it on this object. After ``fetch()`` completes, attribute and relationship access on the peer is @@ -356,6 +357,7 @@ async def fetch(self, timeout: int | None = None) -> None: Args: timeout (int, optional): Overrides the default timeout used when querying the GraphQL API. Specified in seconds. + priority: Override the client-wide request priority for this fetch. When None, the client default is used. Raises: Error: If neither ``id`` nor ``typename`` is set on this related node. @@ -365,7 +367,7 @@ async def fetch(self, timeout: int | None = None) -> None: raise Error("Unable to fetch the peer, id and/or typename are not defined") self._peer = await self._client.get( - kind=self.typename, id=self.id, populate_store=True, branch=self._branch, timeout=timeout + kind=self.typename, id=self.id, populate_store=True, branch=self._branch, timeout=timeout, priority=priority ) @property @@ -443,7 +445,7 @@ def __init__( self._client = client super().__init__(branch=branch, schema=schema, data=data, name=name) - def fetch(self, timeout: int | None = None) -> None: + def fetch(self, timeout: int | None = None, priority: Priority | None = None) -> None: """Fetch the full peer node from the backend and cache it on this object. After ``fetch()`` completes, attribute and relationship access on the peer is @@ -452,6 +454,7 @@ def fetch(self, timeout: int | None = None) -> None: Args: timeout (int, optional): Overrides the default timeout used when querying the GraphQL API. Specified in seconds. + priority: Override the client-wide request priority for this fetch. When None, the client default is used. Raises: Error: If neither ``id`` nor ``typename`` is set on this related node. @@ -461,7 +464,7 @@ def fetch(self, timeout: int | None = None) -> None: raise Error("Unable to fetch the peer, id and/or typename are not defined") self._peer = self._client.get( - kind=self.typename, id=self.id, populate_store=True, branch=self._branch, timeout=timeout + kind=self.typename, id=self.id, populate_store=True, branch=self._branch, timeout=timeout, priority=priority ) @property diff --git a/infrahub_sdk/object_store.py b/infrahub_sdk/object_store.py index 628c06e9a..c770d016a 100644 --- a/infrahub_sdk/object_store.py +++ b/infrahub_sdk/object_store.py @@ -1,6 +1,5 @@ from __future__ import annotations -import copy from typing import TYPE_CHECKING import httpx @@ -42,9 +41,7 @@ def __init__(self, client: InfrahubClient) -> None: async def get(self, identifier: str, tracker: str | None = None) -> str: url = f"{self.client.address}/api/storage/object/{identifier}" - headers = copy.copy(self.client.headers or {}) - if self.client.insert_tracker and tracker: - headers["X-Infrahub-Tracker"] = tracker + headers = self.client._request_headers(tracker=tracker) try: resp = await self.client._get(url=url, headers=headers) @@ -65,9 +62,7 @@ async def get(self, identifier: str, tracker: str | None = None) -> str: async def upload(self, content: str, tracker: str | None = None) -> dict[str, str]: url = f"{self.client.address}/api/storage/upload/content" - headers = copy.copy(self.client.headers or {}) - if self.client.insert_tracker and tracker: - headers["X-Infrahub-Tracker"] = tracker + headers = self.client._request_headers(tracker=tracker) try: resp = await self.client._post(url=url, payload={"content": content}, headers=headers) @@ -94,9 +89,7 @@ async def _get_file(self, url: str, identifier: str, tracker: str | None = None) HTTPStatusError: For other non-2xx HTTP responses. """ - headers = copy.copy(self.client.headers or {}) - if self.client.insert_tracker and tracker: - headers["X-Infrahub-Tracker"] = tracker + headers = self.client._request_headers(tracker=tracker) try: resp = await self.client._get(url=url, headers=headers) @@ -137,9 +130,7 @@ def __init__(self, client: InfrahubClientSync) -> None: def get(self, identifier: str, tracker: str | None = None) -> str: url = f"{self.client.address}/api/storage/object/{identifier}" - headers = copy.copy(self.client.headers or {}) - if self.client.insert_tracker and tracker: - headers["X-Infrahub-Tracker"] = tracker + headers = self.client._request_headers(tracker=tracker) try: resp = self.client._get(url=url, headers=headers) @@ -160,9 +151,7 @@ def get(self, identifier: str, tracker: str | None = None) -> str: def upload(self, content: str, tracker: str | None = None) -> dict[str, str]: url = f"{self.client.address}/api/storage/upload/content" - headers = copy.copy(self.client.headers or {}) - if self.client.insert_tracker and tracker: - headers["X-Infrahub-Tracker"] = tracker + headers = self.client._request_headers(tracker=tracker) try: resp = self.client._post(url=url, payload={"content": content}, headers=headers) @@ -189,9 +178,7 @@ def _get_file(self, url: str, identifier: str, tracker: str | None = None) -> st HTTPStatusError: For other non-2xx HTTP responses. """ - headers = copy.copy(self.client.headers or {}) - if self.client.insert_tracker and tracker: - headers["X-Infrahub-Tracker"] = tracker + headers = self.client._request_headers(tracker=tracker) try: resp = self.client._get(url=url, headers=headers) diff --git a/infrahub_sdk/protocols.py b/infrahub_sdk/protocols.py index c03d689c2..a5a029c22 100644 --- a/infrahub_sdk/protocols.py +++ b/infrahub_sdk/protocols.py @@ -151,6 +151,10 @@ class CoreGroup(CoreNode): children: RelationshipManager +class CoreIPPool(CoreNode): + pass + + class CoreKeyValue(CoreNode): name: String key: String @@ -212,9 +216,13 @@ class CoreTransformation(CoreNode): label: StringOptional description: StringOptional timeout: Integer + fingerprint: StringOptional + dependencies: ListAttributeOptional + dependencies_complete: BooleanOptional query: RelatedNode repository: RelatedNode tags: RelationshipManager + artifact_definitions: RelationshipManager class CoreTriggerRule(CoreNode): @@ -269,6 +277,7 @@ class CoreAccount(LineageOwner, LineageSource, CoreGenericAccount): class CoreAccountGroup(LineageOwner, LineageSource, CoreGroup): + origin: StringOptional roles: RelationshipManager @@ -303,8 +312,11 @@ class CoreArtifactDefinition(CoreTaskTarget): description: StringOptional parameters: JSONAttribute content_type: Enum + fingerprint: StringOptional targets: RelatedNode transformation: RelatedNode + artifacts: RelationshipManager + validators: RelationshipManager class CoreArtifactThread(CoreThread): @@ -336,6 +348,7 @@ class CoreCheckDefinition(CoreTaskTarget): query: RelatedNode targets: RelatedNode tags: RelationshipManager + validators: RelationshipManager class CoreCustomWebhook(CoreWebhook, CoreTaskTarget): @@ -390,9 +403,14 @@ class CoreGeneratorDefinition(CoreTaskTarget): convert_query_response: BooleanOptional execute_in_proposed_change: BooleanOptional execute_after_merge: BooleanOptional + fingerprint: StringOptional + dependencies: ListAttributeOptional + dependencies_complete: BooleanOptional query: RelatedNode repository: RelatedNode targets: RelatedNode + instances: RelationshipManager + validators: RelationshipManager class CoreGeneratorGroup(CoreGroup): @@ -419,6 +437,7 @@ class CoreGraphQLQuery(CoreNode): name: String description: StringOptional query: String + fingerprint: StringOptional variables: JSONAttributeOptional operations: ListAttributeOptional models: ListAttributeOptional @@ -426,6 +445,7 @@ class CoreGraphQLQuery(CoreNode): height: IntegerOptional repository: RelatedNode tags: RelationshipManager + query_groups: RelationshipManager class CoreGraphQLQueryGroup(CoreGroup): @@ -443,14 +463,14 @@ class CoreGroupTriggerRule(CoreTriggerRule): group: RelatedNode -class CoreIPAddressPool(CoreResourcePool, LineageSource): +class CoreIPAddressPool(CoreResourcePool, LineageSource, CoreIPPool): default_address_type: String default_prefix_length: IntegerOptional resources: RelationshipManager ip_namespace: RelatedNode -class CoreIPPrefixPool(CoreResourcePool, LineageSource): +class CoreIPPrefixPool(CoreResourcePool, LineageSource, CoreIPPool): default_prefix_length: IntegerOptional default_member_type: Enum default_prefix_type: StringOptional @@ -736,6 +756,10 @@ class CoreGroupSync(CoreNodeSync): children: RelationshipManagerSync +class CoreIPPoolSync(CoreNodeSync): + pass + + class CoreKeyValueSync(CoreNodeSync): name: String key: String @@ -797,9 +821,13 @@ class CoreTransformationSync(CoreNodeSync): label: StringOptional description: StringOptional timeout: Integer + fingerprint: StringOptional + dependencies: ListAttributeOptional + dependencies_complete: BooleanOptional query: RelatedNodeSync repository: RelatedNodeSync tags: RelationshipManagerSync + artifact_definitions: RelationshipManagerSync class CoreTriggerRuleSync(CoreNodeSync): @@ -854,6 +882,7 @@ class CoreAccountSync(LineageOwnerSync, LineageSourceSync, CoreGenericAccountSyn class CoreAccountGroupSync(LineageOwnerSync, LineageSourceSync, CoreGroupSync): + origin: StringOptional roles: RelationshipManagerSync @@ -888,8 +917,11 @@ class CoreArtifactDefinitionSync(CoreTaskTargetSync): description: StringOptional parameters: JSONAttribute content_type: Enum + fingerprint: StringOptional targets: RelatedNodeSync transformation: RelatedNodeSync + artifacts: RelationshipManagerSync + validators: RelationshipManagerSync class CoreArtifactThreadSync(CoreThreadSync): @@ -921,6 +953,7 @@ class CoreCheckDefinitionSync(CoreTaskTargetSync): query: RelatedNodeSync targets: RelatedNodeSync tags: RelationshipManagerSync + validators: RelationshipManagerSync class CoreCustomWebhookSync(CoreWebhookSync, CoreTaskTargetSync): @@ -975,9 +1008,14 @@ class CoreGeneratorDefinitionSync(CoreTaskTargetSync): convert_query_response: BooleanOptional execute_in_proposed_change: BooleanOptional execute_after_merge: BooleanOptional + fingerprint: StringOptional + dependencies: ListAttributeOptional + dependencies_complete: BooleanOptional query: RelatedNodeSync repository: RelatedNodeSync targets: RelatedNodeSync + instances: RelationshipManagerSync + validators: RelationshipManagerSync class CoreGeneratorGroupSync(CoreGroupSync): @@ -1004,6 +1042,7 @@ class CoreGraphQLQuerySync(CoreNodeSync): name: String description: StringOptional query: String + fingerprint: StringOptional variables: JSONAttributeOptional operations: ListAttributeOptional models: ListAttributeOptional @@ -1011,6 +1050,7 @@ class CoreGraphQLQuerySync(CoreNodeSync): height: IntegerOptional repository: RelatedNodeSync tags: RelationshipManagerSync + query_groups: RelationshipManagerSync class CoreGraphQLQueryGroupSync(CoreGroupSync): @@ -1028,14 +1068,14 @@ class CoreGroupTriggerRuleSync(CoreTriggerRuleSync): group: RelatedNodeSync -class CoreIPAddressPoolSync(CoreResourcePoolSync, LineageSourceSync): +class CoreIPAddressPoolSync(CoreResourcePoolSync, LineageSourceSync, CoreIPPoolSync): default_address_type: String default_prefix_length: IntegerOptional resources: RelationshipManagerSync ip_namespace: RelatedNodeSync -class CoreIPPrefixPoolSync(CoreResourcePoolSync, LineageSourceSync): +class CoreIPPrefixPoolSync(CoreResourcePoolSync, LineageSourceSync, CoreIPPoolSync): default_prefix_length: IntegerOptional default_member_type: Enum default_prefix_type: StringOptional diff --git a/infrahub_sdk/protocols_base.py b/infrahub_sdk/protocols_base.py index 57d4f23fd..7040d1af4 100644 --- a/infrahub_sdk/protocols_base.py +++ b/infrahub_sdk/protocols_base.py @@ -7,7 +7,7 @@ from .context import RequestContext from .node.metadata import NodeMetadata - from .schema import MainSchemaTypes + from .schema import MainSchemaTypesAPI @runtime_checkable @@ -140,6 +140,14 @@ class IPNetworkOptional(Attribute): value: ipaddress.IPv4Network | ipaddress.IPv6Network | None +class IPAddress(Attribute): + value: ipaddress.IPv4Address | ipaddress.IPv6Address + + +class IPAddressOptional(Attribute): + value: ipaddress.IPv4Address | ipaddress.IPv6Address | None + + class Boolean(Attribute): value: bool @@ -173,7 +181,7 @@ class AnyAttributeOptional(Attribute): class CoreNodeBase: - _schema: MainSchemaTypes + _schema: MainSchemaTypesAPI _internal_id: str id: str # NOTE this is incorrect, should be str | None display_label: str | None diff --git a/infrahub_sdk/protocols_generator/constants.py b/infrahub_sdk/protocols_generator/constants.py index 63c3dbb68..282217948 100644 --- a/infrahub_sdk/protocols_generator/constants.py +++ b/infrahub_sdk/protocols_generator/constants.py @@ -17,6 +17,7 @@ "Bandwidth": "Integer", "IPHost": "IPHost", "IPNetwork": "IPNetwork", + "IPAddress": "IPAddress", "Boolean": "Boolean", "Checkbox": "Boolean", "List": "ListAttribute", diff --git a/infrahub_sdk/protocols_generator/template.j2 b/infrahub_sdk/protocols_generator/template.j2 index 43ef24076..a1c9fca53 100644 --- a/infrahub_sdk/protocols_generator/template.j2 +++ b/infrahub_sdk/protocols_generator/template.j2 @@ -31,6 +31,8 @@ if TYPE_CHECKING: IPHostOptional, IPNetwork, IPNetworkOptional, + IPAddress, + IPAddressOptional, JSONAttribute, JSONAttributeOptional, ListAttribute, diff --git a/infrahub_sdk/pytest_plugin/loader.py b/infrahub_sdk/pytest_plugin/loader.py index 7ed0b889b..1d8d34b1d 100644 --- a/infrahub_sdk/pytest_plugin/loader.py +++ b/infrahub_sdk/pytest_plugin/loader.py @@ -66,7 +66,11 @@ def get_resource_config(self, group: InfrahubTestGroup) -> Any | None: return resource_config def collect_group(self, group: InfrahubTestGroup) -> Iterable[pytest.Item]: - """Collect all items for a group.""" + """Collect all items for a group. + + Yields: + pytest.Item: Each collected test item for the group. + """ marker = MARKER_MAPPING[group.resource] resource_config = self.get_resource_config(group) diff --git a/infrahub_sdk/rate_limit.py b/infrahub_sdk/rate_limit.py new file mode 100644 index 000000000..155d4682d --- /dev/null +++ b/infrahub_sdk/rate_limit.py @@ -0,0 +1,169 @@ +from __future__ import annotations + +import asyncio +import logging +import random +import time +from collections.abc import Callable, Coroutine +from datetime import datetime, timezone +from email.utils import parsedate_to_datetime +from typing import TYPE_CHECKING, Any, NoReturn + +import httpx + +from .exceptions import RateLimitError + +if TYPE_CHECKING: + from .types import InfrahubLoggers + +LOGGER = logging.getLogger("infrahub_sdk") + + +class RateLimitRetryHandler: + """Retry logic for HTTP 429 responses. + + The decision methods are pure and stateless (the attempt count is passed in per call), so a + single handler can be shared across concurrent requests. ``send``/``asend`` are the sync and + async I/O drivers that call the sender once per attempt, sleep between retries, and raise + ``RateLimitError`` when the budget is exhausted. + """ + + def __init__( + self, + max_retries: int, + backoff_base: float, + backoff_max: float, + *, + enabled: bool = True, + log: InfrahubLoggers | None = None, + ) -> None: + self.max_retries = max_retries + self.backoff_base = backoff_base + self.backoff_max = backoff_max + self.enabled = enabled + self.log = log or LOGGER + + def parse_retry_after(self, header: str | None, *, now: datetime | None = None) -> float | None: + """Return the ``Retry-After`` wait in seconds, or ``None`` if absent/unparseable. + + Handles both RFC 7231 forms (delta-seconds and HTTP-date); a past date floors to ``0.0``. + """ + value = header.strip() if header is not None else "" + if not value: + return None + + try: + return max(0.0, float(int(value))) + except OverflowError: + return None + except ValueError: + pass # not an integer; try the HTTP-date form below + + try: + parsed = parsedate_to_datetime(value) + except (TypeError, ValueError): + return None + if parsed is None: + return None + + if parsed.tzinfo is None: + parsed = parsed.replace(tzinfo=timezone.utc) + if now is None: + now = datetime.now(timezone.utc) + + delta = (parsed - now).total_seconds() + return max(0.0, delta) + + def compute_backoff(self, attempt: int) -> float: + """Exponential backoff ceiling ``min(backoff_max, backoff_base * 2**attempt)``. + + ``attempt`` is capped at 63 so a very large ``rate_limit_max_retries`` cannot overflow + ``float`` before the clamp applies. + """ + return min(self.backoff_max, self.backoff_base * (2 ** min(attempt, 63))) + + def jittered_delay(self, ceiling: float) -> float: + """Full-jitter delay drawn from ``[0, ceiling]``.""" + return random.uniform(0, ceiling) + + def next_delay(self, attempt: int, retry_after_header: str | None = None, *, now: datetime | None = None) -> float: + """Return the delay in seconds before the next retry. + + Honours a parseable ``Retry-After`` (clamped to ``backoff_max``); otherwise a jittered + exponential backoff (already within ``[0, backoff_max]``). + """ + retry_after = self.parse_retry_after(retry_after_header, now=now) + if retry_after is not None: + return min(retry_after, self.backoff_max) + return self.jittered_delay(self.compute_backoff(attempt)) + + def should_retry(self, attempts_made: int) -> bool: + """Return ``True`` while retries remain (``attempts_made <= max_retries``).""" + return attempts_made <= self.max_retries + + def _raise_exhausted(self, response: httpx.Response, url: str, attempts: int) -> NoReturn: + """Raise ``RateLimitError`` once the retry budget is exhausted. + + Chains the underlying ``httpx.HTTPStatusError`` when one is available. + + Raises: + RateLimitError: Always. + + """ + retry_after = self.parse_retry_after(response.headers.get("Retry-After")) + cause: httpx.HTTPStatusError | None = None + try: + response.raise_for_status() + except httpx.HTTPStatusError as exc: + cause = exc + except RuntimeError: + pass # response has no attached request (custom/fabricated); nothing to chain + raise RateLimitError(url=url, attempts=attempts, retry_after=retry_after) from cause + + async def asend(self, send: Callable[[], Coroutine[Any, Any, httpx.Response]], url: str) -> httpx.Response: + """Send via ``send``, retrying HTTP 429 responses with jittered backoff. + + ``send`` performs one HTTP send per call and MUST yield a fully-readable body each time, + since it is re-invoked per attempt. Honours ``Retry-After`` when present. + + Raises: + RateLimitError: If HTTP 429 responses persist past ``max_retries``. + + """ + if not self.enabled: + return await send() + + attempts = 0 + while True: + response = await send() + attempts += 1 + if response.status_code != 429: + return response + + retry_after_header = response.headers.get("Retry-After") + if not self.should_retry(attempts_made=attempts): + self._raise_exhausted(response=response, url=url, attempts=attempts) + + delay = self.next_delay(attempt=attempts - 1, retry_after_header=retry_after_header) + self.log.warning(f"Rate limited (HTTP 429) on {url}, retry {attempts} in {delay:.2f}s") + await asyncio.sleep(delay) + + def send(self, send: Callable[[], httpx.Response], url: str) -> httpx.Response: + """Synchronous counterpart of :meth:`asend`; see it for the full contract.""" + if not self.enabled: + return send() + + attempts = 0 + while True: + response = send() + attempts += 1 + if response.status_code != 429: + return response + + retry_after_header = response.headers.get("Retry-After") + if not self.should_retry(attempts_made=attempts): + self._raise_exhausted(response=response, url=url, attempts=attempts) + + delay = self.next_delay(attempt=attempts - 1, retry_after_header=retry_after_header) + self.log.warning(f"Rate limited (HTTP 429) on {url}, retry {attempts} in {delay:.2f}s") + time.sleep(delay) diff --git a/infrahub_sdk/schema/__init__.py b/infrahub_sdk/schema/__init__.py index 5343f24cd..37658da37 100644 --- a/infrahub_sdk/schema/__init__.py +++ b/infrahub_sdk/schema/__init__.py @@ -24,6 +24,8 @@ from ..protocols_base import CoreNodeBase from ..queries import SCHEMA_HASH_SYNC_STATUS from .export import RESTRICTED_NAMESPACES, NamespaceExport, SchemaExport, schema_to_export_dict +from .generated.read import InfrahubSchemaRead +from .generated.write import InfrahubSchemaWrite from .main import ( AttributeSchema, AttributeSchemaAPI, @@ -42,6 +44,12 @@ SchemaRootAPI, TemplateSchemaAPI, ) +from .validate import ( + SchemaValidationErrorDetail, + SchemaValidationResult, + SchemaValidationWarningDetail, + validate_schema, +) if TYPE_CHECKING: from ..client import InfrahubClient, InfrahubClientSync, SchemaType, SchemaTypeSync @@ -56,6 +64,8 @@ "BranchSupportType", "GenericSchema", "GenericSchemaAPI", + "InfrahubSchemaRead", + "InfrahubSchemaWrite", "NamespaceExport", "NodeSchema", "NodeSchemaAPI", @@ -67,8 +77,12 @@ "SchemaExport", "SchemaRoot", "SchemaRootAPI", + "SchemaValidationErrorDetail", + "SchemaValidationResult", + "SchemaValidationWarningDetail", "TemplateSchemaAPI", "schema_to_export_dict", + "validate_schema", ] @@ -165,8 +179,19 @@ def _build_export_schemas( ns_map[ns].nodes.append(schema_dict) return SchemaExport(namespaces=ns_map) - def validate(self, data: dict[str, Any]) -> None: - SchemaRoot(**data) + def validate(self, data: dict[str, Any]) -> SchemaValidationResult: + """Validate a schema payload against the generated write contract. + + Returns: + The verdict, carrying a warning for every read-only field the payload sets. + + Raises: + ValueError: When the payload is invalid, joining every field-level message. + + """ + # Delegating to the offline validator keeps this verdict identical to the one + # /api/schema/load reaches, since the server runs the same models. + return validate_schema(schema=data, raise_on_error=True) def validate_data_against_schema(self, schema: MainSchemaTypesAPI, data: dict) -> None: for key in data: @@ -362,7 +387,9 @@ async def load( branch = branch or self.client.default_branch url = f"{self.client.address}/api/schema/load?branch={branch}" response = await self.client._post( - url=url, timeout=max(120, self.client.default_timeout), payload={"schemas": schemas} + url=url, + timeout=max(120, self.client.default_timeout), + payload={"schemas": schemas}, ) if wait_until_converged: @@ -394,7 +421,9 @@ async def check(self, schemas: list[dict], branch: str | None = None) -> tuple[b branch = branch or self.client.default_branch url = f"{self.client.address}/api/schema/check?branch={branch}" response = await self.client._post( - url=url, timeout=max(120, self.client.default_timeout), payload={"schemas": schemas} + url=url, + timeout=max(120, self.client.default_timeout), + payload={"schemas": schemas}, ) if response.status_code == httpx.codes.ACCEPTED: @@ -892,7 +921,9 @@ def load( branch = branch or self.client.default_branch url = f"{self.client.address}/api/schema/load?branch={branch}" response = self.client._post( - url=url, timeout=max(120, self.client.default_timeout), payload={"schemas": schemas} + url=url, + timeout=max(120, self.client.default_timeout), + payload={"schemas": schemas}, ) if wait_until_converged: @@ -924,7 +955,9 @@ def check(self, schemas: list[dict], branch: str | None = None) -> tuple[bool, d branch = branch or self.client.default_branch url = f"{self.client.address}/api/schema/check?branch={branch}" response = self.client._post( - url=url, timeout=max(120, self.client.default_timeout), payload={"schemas": schemas} + url=url, + timeout=max(120, self.client.default_timeout), + payload={"schemas": schemas}, ) if response.status_code == httpx.codes.ACCEPTED: diff --git a/infrahub_sdk/schema/export.py b/infrahub_sdk/schema/export.py index d5a09c775..378f0386d 100644 --- a/infrahub_sdk/schema/export.py +++ b/infrahub_sdk/schema/export.py @@ -49,6 +49,7 @@ def to_dict(self) -> dict[str, dict[str, list[dict[str, Any]]]]: _ATTR_EXPORT_DEFAULTS: dict[str, Any] = { "read_only": False, "optional": False, + "ordered": True, } # Relationship field values that match schema loading defaults — omitted for cleaner output diff --git a/infrahub_sdk/schema/generated/__init__.py b/infrahub_sdk/schema/generated/__init__.py new file mode 100644 index 000000000..3e09a11b0 --- /dev/null +++ b/infrahub_sdk/schema/generated/__init__.py @@ -0,0 +1,4 @@ +# Generated by "invoke backend.generate", do not edit directly +from . import contract, enums, read, write + +__all__ = ["contract", "enums", "read", "write"] diff --git a/infrahub_sdk/schema/generated/contract.py b/infrahub_sdk/schema/generated/contract.py new file mode 100644 index 000000000..c2c5754f3 --- /dev/null +++ b/infrahub_sdk/schema/generated/contract.py @@ -0,0 +1,35 @@ +# Generated by "invoke backend.generate", do not edit directly +"""Read-only fields of the write contract, keyed by generated write class name. + +A field listed here is one the contract knows at that location but the user may not set: +a field the read API returns, the bookkeeping a schema dumped from the internal models +carries, or a field belonging to a sibling variant of a discriminated union. Submitting one +is reported as a warning and the value is dropped, where an extra field that is not listed +is an error. Each entry already includes what the class inherits, so a lookup is by class +name alone. +""" + +READ_ONLY_FIELDS: dict[str, frozenset[str]] = { + "AttributeParametersWrite": frozenset({"id", "state"}), + "AttributeSchemaBaseWrite": frozenset({"inherited"}), + "BaseNodeSchemaWrite": frozenset({"hash", "kind"}), + "ComputedAttributeJinja2Write": frozenset({"id", "state", "transform"}), + "ComputedAttributeTransformPythonWrite": frozenset({"id", "jinja2_template", "state"}), + "ComputedAttributeUserWrite": frozenset({"id", "jinja2_template", "state", "transform"}), + "DropdownChoiceWrite": frozenset({"id", "state"}), + "GenericAttributeWrite": frozenset({"inherited"}), + "GenericSchemaWrite": frozenset({"hash", "kind", "used_by"}), + "InfrahubSchemaWrite": frozenset({"main", "namespaces", "profiles", "templates"}), + "ListAttributeParametersWrite": frozenset({"id", "state"}), + "ListAttributeWrite": frozenset({"inherited"}), + "NodeExtensionWrite": frozenset({"id", "state"}), + "NodeSchemaWrite": frozenset({"hash", "hierarchy", "kind"}), + "NumberAttributeParametersWrite": frozenset({"id", "state"}), + "NumberAttributeWrite": frozenset({"inherited"}), + "NumberPoolAttributeWrite": frozenset({"inherited"}), + "NumberPoolParametersWrite": frozenset({"id", "state"}), + "RelationshipSchemaWrite": frozenset({"hierarchical", "inherited"}), + "SchemaExtensionWrite": frozenset({"id", "state"}), + "TextAttributeParametersWrite": frozenset({"id", "state"}), + "TextAttributeWrite": frozenset({"inherited"}), +} diff --git a/infrahub_sdk/schema/generated/enums.py b/infrahub_sdk/schema/generated/enums.py new file mode 100644 index 000000000..69c6ea2da --- /dev/null +++ b/infrahub_sdk/schema/generated/enums.py @@ -0,0 +1,85 @@ +# Generated by "invoke backend.generate", do not edit directly + +from __future__ import annotations + +from enum import Enum + + +class BranchSupportType(str, Enum): + AWARE = "aware" + AGNOSTIC = "agnostic" + LOCAL = "local" + + +class RelationshipKind(str, Enum): + GENERIC = "Generic" + ATTRIBUTE = "Attribute" + COMPONENT = "Component" + PARENT = "Parent" + GROUP = "Group" + HIERARCHY = "Hierarchy" + PROFILE = "Profile" + TEMPLATE = "Template" + + +class RelationshipCardinality(str, Enum): + ONE = "one" + MANY = "many" + + +class RelationshipDirection(str, Enum): + BIDIR = "bidirectional" + OUTBOUND = "outbound" + INBOUND = "inbound" + + +class RelationshipDeleteBehavior(str, Enum): + NO_ACTION = "no-action" + CASCADE = "cascade" + + +class AllowOverrideType(str, Enum): + NONE = "none" + ANY = "any" + + +class SchemaState(str, Enum): + PRESENT = "present" + ABSENT = "absent" + + +class SchemaAttributeDisplay(str, Enum): + DEFAULT = "default" + EXTRA = "extra" + + +class ComputedAttributeKind(str, Enum): + USER = "User" + JINJA2 = "Jinja2" + TRANSFORM_PYTHON = "TransformPython" + + +class AttributeKind(str, Enum): + ID = "ID" + DROPDOWN = "Dropdown" + TEXT = "Text" + TEXTAREA = "TextArea" + DATETIME = "DateTime" + EMAIL = "Email" + PASSWORD = "Password" + HASHEDPASSWORD = "HashedPassword" + URL = "URL" + FILE = "File" + MAC_ADDRESS = "MacAddress" + COLOR = "Color" + NUMBER = "Number" + NUMBERPOOL = "NumberPool" + BANDWIDTH = "Bandwidth" + IPHOST = "IPHost" + IPNETWORK = "IPNetwork" + IPADDRESS = "IPAddress" + BOOLEAN = "Boolean" + CHECKBOX = "Checkbox" + LIST = "List" + JSON = "JSON" + ANY = "Any" diff --git a/infrahub_sdk/schema/generated/read.py b/infrahub_sdk/schema/generated/read.py new file mode 100644 index 000000000..ecc931430 --- /dev/null +++ b/infrahub_sdk/schema/generated/read.py @@ -0,0 +1,603 @@ +# Generated by "invoke backend.generate", do not edit directly + +from __future__ import annotations + +from typing import Annotated, Any, Literal + +from pydantic import BaseModel, ConfigDict, Field, computed_field + +from .enums import ( + AllowOverrideType, + AttributeKind, + BranchSupportType, + ComputedAttributeKind, + RelationshipCardinality, + RelationshipDeleteBehavior, + RelationshipDirection, + RelationshipKind, + SchemaAttributeDisplay, + SchemaState, +) + + +class AttributeParametersRead(BaseModel): + model_config = ConfigDict(extra="ignore", use_enum_values=True) + + +class ListAttributeParametersRead(AttributeParametersRead): + model_config = ConfigDict(extra="ignore", use_enum_values=True) + regex: str | None = Field( + default=None, + description="Regular expression that each list item value must match if defined", + ) + + +class TextAttributeParametersRead(AttributeParametersRead): + model_config = ConfigDict(extra="ignore", use_enum_values=True) + regex: str | None = Field( + default=None, + description="Regular expression that attribute value must match if defined", + ) + min_length: int | None = Field( + default=None, + description="Set a minimum number of characters allowed.", + ) + max_length: int | None = Field( + default=None, + description="Set a maximum number of characters allowed.", + ) + + +class NumberAttributeParametersRead(AttributeParametersRead): + model_config = ConfigDict(extra="ignore", use_enum_values=True) + min_value: int | None = Field( + default=None, + description="Set a minimum value allowed.", + ) + max_value: int | None = Field( + default=None, + description="Set a maximum value allowed.", + ) + excluded_values: str | None = Field( + default=None, + description="List of values or range of values not allowed for the attribute, format is: '100,150-200,280,300-400'", + pattern=r"^(\d+(?:-\d+)?)(?:,\d+(?:-\d+)?)*$", + ) + + +class NumberPoolParametersRead(AttributeParametersRead): + model_config = ConfigDict(extra="ignore", use_enum_values=True) + end_range: int = Field( + default=9223372036854775807, + description="End range for numbers for the associated NumberPool", + ) + start_range: int = Field( + default=1, + description="Start range for numbers for the associated NumberPool", + ) + number_pool_id: str | None = Field( + default=None, + description="The ID of the numberpool associated with this attribute. Only set after the number pool has been provisioned.", + ) + + +class DropdownChoiceRead(BaseModel): + model_config = ConfigDict(extra="ignore", use_enum_values=True) + name: str = Field( + ..., + description="Name of the choice, must be unique within the dropdown.", + ) + description: str | None = Field( + default=None, + description="Description of the choice.", + ) + color: str | None = Field( + default=None, + description="Color of the choice, must be a valid HTML color code.", + pattern=r"#[0-9a-fA-F]{6}\b", + ) + label: str | None = Field( + default=None, + description="Human friendly representation of the choice.", + ) + + +class ComputedAttributeUserRead(BaseModel): + model_config = ConfigDict(extra="ignore", use_enum_values=True) + kind: Literal[ComputedAttributeKind.USER] = Field( + ..., + description="Defines how the value of the attribute is computed.", + ) + + +class ComputedAttributeJinja2Read(BaseModel): + model_config = ConfigDict(extra="ignore", use_enum_values=True) + kind: Literal[ComputedAttributeKind.JINJA2] = Field( + ..., + description="Defines how the value of the attribute is computed.", + ) + jinja2_template: str = Field( + ..., + description="Jinja2 template used to compute the value, required when kind is Jinja2.", + ) + + +class ComputedAttributeTransformPythonRead(BaseModel): + model_config = ConfigDict(extra="ignore", use_enum_values=True) + kind: Literal[ComputedAttributeKind.TRANSFORM_PYTHON] = Field( + ..., + description="Defines how the value of the attribute is computed.", + ) + transform: str = Field( + ..., + description="Python transform name or ID, required when kind is TransformPython.", + ) + + +class AttributeSchemaBaseRead(BaseModel): + model_config = ConfigDict(extra="ignore", use_enum_values=True) + id: str | None = Field( + default=None, + description="The ID of the attribute", + ) + name: str = Field( + ..., + description="Attribute name, must be unique within a model and must be all lowercase.", + pattern=r"^[a-z0-9\_]+$", + min_length=3, + max_length=64, + ) + kind: AttributeKind = Field( + ..., + description="Defines the type of the attribute.", + ) + enum: list | None = Field( + default=None, + description="Define a list of valid values for the attribute.", + ) + computed_attribute: ComputedAttributeRead | None = Field( + default=None, + description="Defines how the value of this attribute will be populated.", + ) + choices: list[DropdownChoiceRead] | None = Field( + default=None, + description="Define a list of valid choices for a dropdown attribute.", + ) + regex: str | None = Field( + default=None, + description="Regex uses to limit the characters allowed in for the attributes. (deprecated: please use parameters.regex instead)", + ) + max_length: int | None = Field( + default=None, + description="Set a maximum number of characters allowed for a given attribute. (deprecated: please use parameters.max_length instead)", + ) + min_length: int | None = Field( + default=None, + description="Set a minimum number of characters allowed for a given attribute. (deprecated: please use parameters.min_length instead)", + ) + label: str | None = Field( + default=None, + description="Human friendly representation of the name. Will be autogenerated if not provided", + max_length=64, + ) + description: str | None = Field( + default=None, + description="Short description of the attribute.", + max_length=128, + ) + read_only: bool = Field( + default=False, + description="Set the attribute as Read-Only, users won't be able to change its value. Mainly relevant for internal object.", + ) + unique: bool = Field( + default=False, + description="Indicate if the value of this attribute must be unique in the database for a given model.", + ) + optional: bool = Field( + default=False, + description="Indicate if this attribute is mandatory or optional.", + ) + branch: BranchSupportType | None = Field( + default=None, + description="Type of branch support for the attribute, if not defined it will be inherited from the node.", + ) + order_weight: int | None = Field( + default=None, + description="Number used to order the attribute in the frontend (table and view). Lowest value will be ordered first.", + ) + ordered: bool = Field( + default=True, + description="Whether element order is significant. When False, reordering a List or JSON-array attribute is not a merge/rebase conflict.", + ) + default_value: Any | None = Field( + default=None, + description="Default value of the attribute.", + ) + inherited: bool = Field( + default=False, + description="Internal value to indicate if the attribute was inherited from a Generic node.", + ) + state: SchemaState = Field( + default=SchemaState.PRESENT, + description="Expected state of the attribute after loading the schema", + ) + allow_override: AllowOverrideType = Field( + default=AllowOverrideType.ANY, + description="Type of allowed override for the attribute.", + ) + deprecation: str | None = Field( + default=None, + description="Mark attribute as deprecated and provide a user-friendly message to display", + max_length=128, + ) + display: SchemaAttributeDisplay = Field( + default=SchemaAttributeDisplay.DEFAULT, + description="Controls where the attribute is displayed. 'default' shows in the main view, 'extra' shows in an expanded/secondary section.", + ) + + +class TextAttributeRead(AttributeSchemaBaseRead): + model_config = ConfigDict(extra="ignore", use_enum_values=True) + kind: Literal[AttributeKind.TEXT, AttributeKind.TEXTAREA] = Field( + ..., + description="Defines the type of the attribute.", + ) + parameters: TextAttributeParametersRead | None = Field( + default=None, + description="Extra parameters specific to this kind of attribute", + ) + + +class NumberAttributeRead(AttributeSchemaBaseRead): + model_config = ConfigDict(extra="ignore", use_enum_values=True) + kind: Literal[AttributeKind.NUMBER] = Field( + ..., + description="Defines the type of the attribute.", + ) + parameters: NumberAttributeParametersRead | None = Field( + default=None, + description="Extra parameters specific to this kind of attribute", + ) + + +class ListAttributeRead(AttributeSchemaBaseRead): + model_config = ConfigDict(extra="ignore", use_enum_values=True) + kind: Literal[AttributeKind.LIST] = Field( + ..., + description="Defines the type of the attribute.", + ) + parameters: ListAttributeParametersRead | None = Field( + default=None, + description="Extra parameters specific to this kind of attribute", + ) + + +class NumberPoolAttributeRead(AttributeSchemaBaseRead): + model_config = ConfigDict(extra="ignore", use_enum_values=True) + kind: Literal[AttributeKind.NUMBERPOOL] = Field( + ..., + description="Defines the type of the attribute.", + ) + parameters: NumberPoolParametersRead | None = Field( + default=None, + description="Extra parameters specific to this kind of attribute", + ) + + +class GenericAttributeRead(AttributeSchemaBaseRead): + model_config = ConfigDict(extra="ignore", use_enum_values=True) + kind: Literal[ + AttributeKind.ID, + AttributeKind.DROPDOWN, + AttributeKind.DATETIME, + AttributeKind.EMAIL, + AttributeKind.PASSWORD, + AttributeKind.HASHEDPASSWORD, + AttributeKind.URL, + AttributeKind.FILE, + AttributeKind.MAC_ADDRESS, + AttributeKind.COLOR, + AttributeKind.BANDWIDTH, + AttributeKind.IPHOST, + AttributeKind.IPNETWORK, + AttributeKind.IPADDRESS, + AttributeKind.BOOLEAN, + AttributeKind.CHECKBOX, + AttributeKind.JSON, + AttributeKind.ANY, + ] = Field( + ..., + description="Defines the type of the attribute.", + ) + parameters: AttributeParametersRead | None = Field( + default=None, + description="Extra parameters specific to this kind of attribute", + ) + + +ComputedAttributeRead = Annotated[ + ComputedAttributeUserRead | ComputedAttributeJinja2Read | ComputedAttributeTransformPythonRead, + Field(discriminator="kind"), +] + +AttributeSchemaRead = Annotated[ + TextAttributeRead | NumberAttributeRead | ListAttributeRead | NumberPoolAttributeRead | GenericAttributeRead, + Field(discriminator="kind"), +] + + +class RelationshipSchemaRead(BaseModel): + model_config = ConfigDict(extra="ignore", use_enum_values=True) + id: str | None = Field( + default=None, + description="The ID of the relationship schema", + ) + name: str = Field( + ..., + description="Relationship name, must be unique within a model and must be all lowercase.", + pattern=r"^[a-z0-9\_]+$", + min_length=3, + max_length=64, + ) + peer: str = Field( + ..., + description="Type (kind) of objects supported on the other end of the relationship.", + pattern=r"^[A-Z][a-zA-Z0-9]+$", + ) + kind: RelationshipKind = Field( + default=RelationshipKind.GENERIC, + description="Defines the type of the relationship.", + ) + label: str | None = Field( + default=None, + description="Human friendly representation of the name. Will be autogenerated if not provided", + max_length=64, + ) + description: str | None = Field( + default=None, + description="Short description of the relationship.", + max_length=128, + ) + identifier: str | None = Field( + default=None, + description="Unique identifier of the relationship within a model, identifiers must match to traverse a relationship on both direction.", + pattern=r"^[a-z0-9\_]+$", + max_length=128, + ) + cardinality: RelationshipCardinality = Field( + default=RelationshipCardinality.MANY, + description="Defines how many objects are expected on the other side of the relationship.", + ) + min_count: int = Field( + default=0, + description="Defines the minimum objects allowed on the other side of the relationship.", + ) + max_count: int = Field( + default=0, + description="Defines the maximum objects allowed on the other side of the relationship.", + ) + common_parent: str | None = Field( + default=None, + description="Name of a parent relationship on the peer schema that must share the same related object with the object's parent.", + ) + common_relatives: list[str] | None = Field( + default=None, + description="List of relationship names on the peer schema for which all objects must share the same set of peers.", + ) + order_weight: int | None = Field( + default=None, + description="Number used to order the relationship in the frontend (table and view). Lowest value will be ordered first.", + ) + optional: bool = Field( + default=True, + description="Indicate if this relationship is mandatory or optional.", + ) + branch: BranchSupportType | None = Field( + default=None, + description="Type of branch support for the relationship. If not defined, it will be determined based on both peers.", + ) + inherited: bool = Field( + default=False, + description="Internal value to indicate if the relationship was inherited from a Generic node.", + ) + direction: RelationshipDirection = Field( + default=RelationshipDirection.BIDIR, + description="Defines the direction of the relationship, Unidirectional relationship are required when the same model is on both side.", + ) + hierarchical: str | None = Field( + default=None, + description="Internal attribute to track the type of hierarchy this relationship is part of, must match a valid Generic Kind", + ) + state: SchemaState = Field( + default=SchemaState.PRESENT, + description="Expected state of the relationship after loading the schema", + ) + on_delete: RelationshipDeleteBehavior | None = Field( + default=None, + description="Default is no-action. If cascade, related node(s) are deleted when this node is deleted.", + ) + allow_override: AllowOverrideType = Field( + default=AllowOverrideType.ANY, + description="Type of allowed override for the relationship.", + ) + read_only: bool = Field( + default=False, + description="Set the relationship as read-only, users won't be able to change its value.", + ) + deprecation: str | None = Field( + default=None, + description="Mark relationship as deprecated and provide a user-friendly message to display", + max_length=128, + ) + display: SchemaAttributeDisplay = Field( + default=SchemaAttributeDisplay.DEFAULT, + description="Controls where the relationship is displayed. 'default' shows in the main view, 'extra' shows in an expanded/secondary section.", + ) + + +class BaseNodeSchemaRead(BaseModel): + model_config = ConfigDict(extra="ignore", use_enum_values=True) + id: str | None = Field( + default=None, + description="The ID of the node", + ) + name: str = Field( + ..., + description="Node name, must be unique within a namespace and must start with an uppercase letter.", + pattern=r"^[A-Z][a-zA-Z0-9]+$", + min_length=2, + max_length=32, + ) + namespace: str = Field( + ..., + description="Node Namespace, Namespaces are used to organize models into logical groups and to prevent name collisions.", + pattern=r"^[A-Z][a-z0-9]+$", + min_length=3, + max_length=64, + ) + description: str | None = Field( + default=None, + description="Short description of the model, will be visible in the frontend.", + max_length=128, + ) + label: str | None = Field( + default=None, + description="Human friendly representation of the name/kind", + max_length=64, + ) + branch: BranchSupportType = Field( + default=BranchSupportType.AWARE, + description="Type of branch support for the model.", + ) + default_filter: str | None = Field( + default=None, + description="Default filter used to search for a node in addition to its ID. (deprecated: please use human_friendly_id instead)", + pattern=r"^[a-z0-9\_]*$", + ) + human_friendly_id: list[str] | None = Field( + default=None, + description="Human friendly and unique identifier for the object.", + ) + display_label: str | None = Field( + default=None, + description="Attribute or Jinja2 template to use to generate the display label", + ) + display_labels: list[str] | None = Field( + default=None, + description="List of attributes to use to generate the display label (deprecated)", + ) + include_in_menu: bool | None = Field( + default=None, + description="Defines if objects of this kind should be included in the menu.", + ) + menu_placement: str | None = Field( + default=None, + description="Defines where in the menu this object should be placed.", + ) + icon: str | None = Field( + default=None, + description="Defines the icon to use in the menu. Must be a valid value from the MDI library https://icon-sets.iconify.design/mdi/", + ) + order_by: list[str] | None = Field( + default=None, + description="List of entries to order results by. Supports attributes, relationship attributes, and node_metadata with __asc/__desc.", + ) + uniqueness_constraints: list[list[str]] | None = Field( + default=None, + description="List of multi-element uniqueness constraints that can combine relationships and attributes", + ) + documentation: str | None = Field( + default=None, + description="Link to a documentation associated with this object, can be internal or external.", + ) + state: SchemaState = Field( + default=SchemaState.PRESENT, + description="Expected state of the node/generic after loading the schema", + ) + attributes: list[AttributeSchemaRead] = Field( + default_factory=list, + description="Node attributes", + ) + relationships: list[RelationshipSchemaRead] = Field( + default_factory=list, + description="Node Relationships", + ) + hash: str | None = Field( + default=None, + description="Hash of the node computed by the server.", + ) + + @computed_field + @property + def kind(self) -> str: + return f"{self.namespace}{self.name}" + + +class NodeSchemaRead(BaseNodeSchemaRead): + model_config = ConfigDict(extra="ignore", use_enum_values=True) + inherit_from: list[str] = Field( + default_factory=list, + description="List of Generic Kind that this node is inheriting from", + ) + generate_profile: bool = Field( + default=True, + description="Indicate if a profile schema should be generated for this schema", + ) + generate_template: bool = Field( + default=False, + description="Indicate if an object template schema should be generated for this schema", + ) + hierarchy: str | None = Field( + default=None, + description="Internal value to track the name of the Hierarchy, must match the name of a Generic supporting hierarchical mode", + ) + parent: str | None = Field( + default=None, + description="Expected Kind for the parent node in a Hierarchy, default to the main generic defined if not defined.", + ) + children: str | None = Field( + default=None, + description="Expected Kind for the children nodes in a Hierarchy, default to the main generic defined if not defined.", + ) + + +class GenericSchemaRead(BaseNodeSchemaRead): + model_config = ConfigDict(extra="ignore", use_enum_values=True) + hierarchical: bool = Field( + default=False, + description="Defines if the Generic support the hierarchical mode.", + ) + generate_profile: bool = Field( + default=True, + description="Indicate if a profile schema should be generated for this schema", + ) + used_by: list[str] = Field( + default_factory=list, + description="List of Nodes that are referencing this Generic", + ) + restricted_namespaces: list[str] | None = Field( + default=None, + description="Nodes inheriting from this Generic schema must belong to one of the listed namespaces", + ) + + +class ProfileSchemaRead(BaseNodeSchemaRead): + model_config = ConfigDict(extra="ignore", use_enum_values=True) + inherit_from: list[str] = Field( + default_factory=list, + description="List of Generic Kind that this profile is inheriting from", + ) + + +class TemplateSchemaRead(BaseNodeSchemaRead): + model_config = ConfigDict(extra="ignore", use_enum_values=True) + inherit_from: list[str] = Field( + default_factory=list, + description="List of Generic Kind that this template is inheriting from", + ) + + +class InfrahubSchemaRead(BaseModel): + model_config = ConfigDict(extra="ignore", use_enum_values=True) + nodes: list[NodeSchemaRead] = Field(default_factory=list) + generics: list[GenericSchemaRead] = Field(default_factory=list) diff --git a/infrahub_sdk/schema/generated/write.py b/infrahub_sdk/schema/generated/write.py new file mode 100644 index 000000000..a834ea442 --- /dev/null +++ b/infrahub_sdk/schema/generated/write.py @@ -0,0 +1,588 @@ +# Generated by "invoke backend.generate", do not edit directly + +from __future__ import annotations + +from typing import Annotated, Any, Literal + +from pydantic import BaseModel, ConfigDict, Field + +from .enums import ( + AllowOverrideType, + AttributeKind, + BranchSupportType, + ComputedAttributeKind, + RelationshipCardinality, + RelationshipDeleteBehavior, + RelationshipDirection, + RelationshipKind, + SchemaAttributeDisplay, + SchemaState, +) + + +class AttributeParametersWrite(BaseModel): + model_config = ConfigDict(extra="ignore", use_enum_values=True) + + +class ListAttributeParametersWrite(AttributeParametersWrite): + model_config = ConfigDict(extra="ignore", use_enum_values=True) + regex: str | None = Field( + default=None, + description="Regular expression that each list item value must match if defined", + ) + + +class TextAttributeParametersWrite(AttributeParametersWrite): + model_config = ConfigDict(extra="ignore", use_enum_values=True) + regex: str | None = Field( + default=None, + description="Regular expression that attribute value must match if defined", + ) + min_length: int | None = Field( + default=None, + description="Set a minimum number of characters allowed.", + ) + max_length: int | None = Field( + default=None, + description="Set a maximum number of characters allowed.", + ) + + +class NumberAttributeParametersWrite(AttributeParametersWrite): + model_config = ConfigDict(extra="ignore", use_enum_values=True) + min_value: int | None = Field( + default=None, + description="Set a minimum value allowed.", + ) + max_value: int | None = Field( + default=None, + description="Set a maximum value allowed.", + ) + excluded_values: str | None = Field( + default=None, + description="List of values or range of values not allowed for the attribute, format is: '100,150-200,280,300-400'", + pattern=r"^(\d+(?:-\d+)?)(?:,\d+(?:-\d+)?)*$", + ) + + +class NumberPoolParametersWrite(AttributeParametersWrite): + model_config = ConfigDict(extra="ignore", use_enum_values=True) + end_range: int = Field( + default=9223372036854775807, + description="End range for numbers for the associated NumberPool", + ) + start_range: int = Field( + default=1, + description="Start range for numbers for the associated NumberPool", + ) + number_pool_id: str | None = Field( + default=None, + description="The ID of the numberpool associated with this attribute. Only set after the number pool has been provisioned.", + ) + + +class DropdownChoiceWrite(BaseModel): + model_config = ConfigDict(extra="ignore", use_enum_values=True) + name: str = Field( + ..., + description="Name of the choice, must be unique within the dropdown.", + ) + description: str | None = Field( + default=None, + description="Description of the choice.", + ) + color: str | None = Field( + default=None, + description="Color of the choice, must be a valid HTML color code.", + pattern=r"#[0-9a-fA-F]{6}\b", + ) + label: str | None = Field( + default=None, + description="Human friendly representation of the choice.", + ) + + +class ComputedAttributeUserWrite(BaseModel): + model_config = ConfigDict(extra="ignore", use_enum_values=True) + kind: Literal[ComputedAttributeKind.USER] = Field( + ..., + description="Defines how the value of the attribute is computed.", + ) + + +class ComputedAttributeJinja2Write(BaseModel): + model_config = ConfigDict(extra="ignore", use_enum_values=True) + kind: Literal[ComputedAttributeKind.JINJA2] = Field( + ..., + description="Defines how the value of the attribute is computed.", + ) + jinja2_template: str = Field( + ..., + description="Jinja2 template used to compute the value, required when kind is Jinja2.", + ) + + +class ComputedAttributeTransformPythonWrite(BaseModel): + model_config = ConfigDict(extra="ignore", use_enum_values=True) + kind: Literal[ComputedAttributeKind.TRANSFORM_PYTHON] = Field( + ..., + description="Defines how the value of the attribute is computed.", + ) + transform: str = Field( + ..., + description="Python transform name or ID, required when kind is TransformPython.", + ) + + +class AttributeSchemaBaseWrite(BaseModel): + model_config = ConfigDict(extra="ignore", use_enum_values=True) + id: str | None = Field( + default=None, + description="The ID of the attribute", + ) + name: str = Field( + ..., + description="Attribute name, must be unique within a model and must be all lowercase.", + pattern=r"^[a-z0-9\_]+$", + min_length=3, + max_length=64, + ) + kind: AttributeKind = Field( + ..., + description="Defines the type of the attribute.", + ) + enum: list | None = Field( + default=None, + description="Define a list of valid values for the attribute.", + ) + computed_attribute: ComputedAttributeWrite | None = Field( + default=None, + description="Defines how the value of this attribute will be populated.", + ) + choices: list[DropdownChoiceWrite] | None = Field( + default=None, + description="Define a list of valid choices for a dropdown attribute.", + ) + regex: str | None = Field( + default=None, + description="Regex uses to limit the characters allowed in for the attributes. (deprecated: please use parameters.regex instead)", + ) + max_length: int | None = Field( + default=None, + description="Set a maximum number of characters allowed for a given attribute. (deprecated: please use parameters.max_length instead)", + ) + min_length: int | None = Field( + default=None, + description="Set a minimum number of characters allowed for a given attribute. (deprecated: please use parameters.min_length instead)", + ) + label: str | None = Field( + default=None, + description="Human friendly representation of the name. Will be autogenerated if not provided", + max_length=64, + ) + description: str | None = Field( + default=None, + description="Short description of the attribute.", + max_length=128, + ) + read_only: bool = Field( + default=False, + description="Set the attribute as Read-Only, users won't be able to change its value. Mainly relevant for internal object.", + ) + unique: bool = Field( + default=False, + description="Indicate if the value of this attribute must be unique in the database for a given model.", + ) + optional: bool = Field( + default=False, + description="Indicate if this attribute is mandatory or optional.", + ) + branch: BranchSupportType | None = Field( + default=None, + description="Type of branch support for the attribute, if not defined it will be inherited from the node.", + ) + order_weight: int | None = Field( + default=None, + description="Number used to order the attribute in the frontend (table and view). Lowest value will be ordered first.", + ) + ordered: bool = Field( + default=True, + description="Whether element order is significant. When False, reordering a List or JSON-array attribute is not a merge/rebase conflict.", + ) + default_value: Any | None = Field( + default=None, + description="Default value of the attribute.", + ) + state: SchemaState = Field( + default=SchemaState.PRESENT, + description="Expected state of the attribute after loading the schema", + ) + allow_override: AllowOverrideType = Field( + default=AllowOverrideType.ANY, + description="Type of allowed override for the attribute.", + ) + deprecation: str | None = Field( + default=None, + description="Mark attribute as deprecated and provide a user-friendly message to display", + max_length=128, + ) + display: SchemaAttributeDisplay = Field( + default=SchemaAttributeDisplay.DEFAULT, + description="Controls where the attribute is displayed. 'default' shows in the main view, 'extra' shows in an expanded/secondary section.", + ) + + +class TextAttributeWrite(AttributeSchemaBaseWrite): + model_config = ConfigDict(extra="ignore", use_enum_values=True) + kind: Literal[AttributeKind.TEXT, AttributeKind.TEXTAREA] = Field( + ..., + description="Defines the type of the attribute.", + ) + parameters: TextAttributeParametersWrite | None = Field( + default=None, + description="Extra parameters specific to this kind of attribute", + ) + + +class NumberAttributeWrite(AttributeSchemaBaseWrite): + model_config = ConfigDict(extra="ignore", use_enum_values=True) + kind: Literal[AttributeKind.NUMBER] = Field( + ..., + description="Defines the type of the attribute.", + ) + parameters: NumberAttributeParametersWrite | None = Field( + default=None, + description="Extra parameters specific to this kind of attribute", + ) + + +class ListAttributeWrite(AttributeSchemaBaseWrite): + model_config = ConfigDict(extra="ignore", use_enum_values=True) + kind: Literal[AttributeKind.LIST] = Field( + ..., + description="Defines the type of the attribute.", + ) + parameters: ListAttributeParametersWrite | None = Field( + default=None, + description="Extra parameters specific to this kind of attribute", + ) + + +class NumberPoolAttributeWrite(AttributeSchemaBaseWrite): + model_config = ConfigDict(extra="ignore", use_enum_values=True) + kind: Literal[AttributeKind.NUMBERPOOL] = Field( + ..., + description="Defines the type of the attribute.", + ) + parameters: NumberPoolParametersWrite | None = Field( + default=None, + description="Extra parameters specific to this kind of attribute", + ) + + +class GenericAttributeWrite(AttributeSchemaBaseWrite): + model_config = ConfigDict(extra="ignore", use_enum_values=True) + kind: Literal[ + AttributeKind.ID, + AttributeKind.DROPDOWN, + AttributeKind.DATETIME, + AttributeKind.EMAIL, + AttributeKind.PASSWORD, + AttributeKind.HASHEDPASSWORD, + AttributeKind.URL, + AttributeKind.FILE, + AttributeKind.MAC_ADDRESS, + AttributeKind.COLOR, + AttributeKind.BANDWIDTH, + AttributeKind.IPHOST, + AttributeKind.IPNETWORK, + AttributeKind.IPADDRESS, + AttributeKind.BOOLEAN, + AttributeKind.CHECKBOX, + AttributeKind.JSON, + AttributeKind.ANY, + ] = Field( + ..., + description="Defines the type of the attribute.", + ) + parameters: AttributeParametersWrite | None = Field( + default=None, + description="Extra parameters specific to this kind of attribute", + ) + + +ComputedAttributeWrite = Annotated[ + ComputedAttributeUserWrite | ComputedAttributeJinja2Write | ComputedAttributeTransformPythonWrite, + Field(discriminator="kind"), +] + +AttributeSchemaWrite = Annotated[ + TextAttributeWrite | NumberAttributeWrite | ListAttributeWrite | NumberPoolAttributeWrite | GenericAttributeWrite, + Field(discriminator="kind"), +] + + +class RelationshipSchemaWrite(BaseModel): + model_config = ConfigDict(extra="ignore", use_enum_values=True) + id: str | None = Field( + default=None, + description="The ID of the relationship schema", + ) + name: str = Field( + ..., + description="Relationship name, must be unique within a model and must be all lowercase.", + pattern=r"^[a-z0-9\_]+$", + min_length=3, + max_length=64, + ) + peer: str = Field( + ..., + description="Type (kind) of objects supported on the other end of the relationship.", + pattern=r"^[A-Z][a-zA-Z0-9]+$", + ) + kind: RelationshipKind = Field( + default=RelationshipKind.GENERIC, + description="Defines the type of the relationship.", + ) + label: str | None = Field( + default=None, + description="Human friendly representation of the name. Will be autogenerated if not provided", + max_length=64, + ) + description: str | None = Field( + default=None, + description="Short description of the relationship.", + max_length=128, + ) + identifier: str | None = Field( + default=None, + description="Unique identifier of the relationship within a model, identifiers must match to traverse a relationship on both direction.", + pattern=r"^[a-z0-9\_]+$", + max_length=128, + ) + cardinality: RelationshipCardinality = Field( + default=RelationshipCardinality.MANY, + description="Defines how many objects are expected on the other side of the relationship.", + ) + min_count: int = Field( + default=0, + description="Defines the minimum objects allowed on the other side of the relationship.", + ) + max_count: int = Field( + default=0, + description="Defines the maximum objects allowed on the other side of the relationship.", + ) + common_parent: str | None = Field( + default=None, + description="Name of a parent relationship on the peer schema that must share the same related object with the object's parent.", + ) + common_relatives: list[str] | None = Field( + default=None, + description="List of relationship names on the peer schema for which all objects must share the same set of peers.", + ) + order_weight: int | None = Field( + default=None, + description="Number used to order the relationship in the frontend (table and view). Lowest value will be ordered first.", + ) + optional: bool = Field( + default=True, + description="Indicate if this relationship is mandatory or optional.", + ) + branch: BranchSupportType | None = Field( + default=None, + description="Type of branch support for the relationship. If not defined, it will be determined based on both peers.", + ) + direction: RelationshipDirection = Field( + default=RelationshipDirection.BIDIR, + description="Defines the direction of the relationship, Unidirectional relationship are required when the same model is on both side.", + ) + state: SchemaState = Field( + default=SchemaState.PRESENT, + description="Expected state of the relationship after loading the schema", + ) + on_delete: RelationshipDeleteBehavior | None = Field( + default=None, + description="Default is no-action. If cascade, related node(s) are deleted when this node is deleted.", + ) + allow_override: AllowOverrideType = Field( + default=AllowOverrideType.ANY, + description="Type of allowed override for the relationship.", + ) + read_only: bool = Field( + default=False, + description="Set the relationship as read-only, users won't be able to change its value.", + ) + deprecation: str | None = Field( + default=None, + description="Mark relationship as deprecated and provide a user-friendly message to display", + max_length=128, + ) + display: SchemaAttributeDisplay = Field( + default=SchemaAttributeDisplay.DEFAULT, + description="Controls where the relationship is displayed. 'default' shows in the main view, 'extra' shows in an expanded/secondary section.", + ) + + +class BaseNodeSchemaWrite(BaseModel): + model_config = ConfigDict(extra="ignore", use_enum_values=True) + id: str | None = Field( + default=None, + description="The ID of the node", + ) + name: str = Field( + ..., + description="Node name, must be unique within a namespace and must start with an uppercase letter.", + pattern=r"^[A-Z][a-zA-Z0-9]+$", + min_length=2, + max_length=32, + ) + namespace: str = Field( + ..., + description="Node Namespace, Namespaces are used to organize models into logical groups and to prevent name collisions.", + pattern=r"^[A-Z][a-z0-9]+$", + min_length=3, + max_length=64, + ) + description: str | None = Field( + default=None, + description="Short description of the model, will be visible in the frontend.", + max_length=128, + ) + label: str | None = Field( + default=None, + description="Human friendly representation of the name/kind", + max_length=64, + ) + branch: BranchSupportType = Field( + default=BranchSupportType.AWARE, + description="Type of branch support for the model.", + ) + default_filter: str | None = Field( + default=None, + description="Default filter used to search for a node in addition to its ID. (deprecated: please use human_friendly_id instead)", + pattern=r"^[a-z0-9\_]*$", + ) + human_friendly_id: list[str] | None = Field( + default=None, + description="Human friendly and unique identifier for the object.", + ) + display_label: str | None = Field( + default=None, + description="Attribute or Jinja2 template to use to generate the display label", + ) + display_labels: list[str] | None = Field( + default=None, + description="List of attributes to use to generate the display label (deprecated)", + ) + include_in_menu: bool | None = Field( + default=None, + description="Defines if objects of this kind should be included in the menu.", + ) + menu_placement: str | None = Field( + default=None, + description="Defines where in the menu this object should be placed.", + ) + icon: str | None = Field( + default=None, + description="Defines the icon to use in the menu. Must be a valid value from the MDI library https://icon-sets.iconify.design/mdi/", + ) + order_by: list[str] | None = Field( + default=None, + description="List of entries to order results by. Supports attributes, relationship attributes, and node_metadata with __asc/__desc.", + ) + uniqueness_constraints: list[list[str]] | None = Field( + default=None, + description="List of multi-element uniqueness constraints that can combine relationships and attributes", + ) + documentation: str | None = Field( + default=None, + description="Link to a documentation associated with this object, can be internal or external.", + ) + state: SchemaState = Field( + default=SchemaState.PRESENT, + description="Expected state of the node/generic after loading the schema", + ) + attributes: list[AttributeSchemaWrite] = Field( + default_factory=list, + description="Node attributes", + ) + relationships: list[RelationshipSchemaWrite] = Field( + default_factory=list, + description="Node Relationships", + ) + + @property + def kind(self) -> str: + return f"{self.namespace}{self.name}" + + +class NodeSchemaWrite(BaseNodeSchemaWrite): + model_config = ConfigDict(extra="ignore", use_enum_values=True) + inherit_from: list[str] = Field( + default_factory=list, + description="List of Generic Kind that this node is inheriting from", + ) + generate_profile: bool = Field( + default=True, + description="Indicate if a profile schema should be generated for this schema", + ) + generate_template: bool = Field( + default=False, + description="Indicate if an object template schema should be generated for this schema", + ) + parent: str | None = Field( + default=None, + description="Expected Kind for the parent node in a Hierarchy, default to the main generic defined if not defined.", + ) + children: str | None = Field( + default=None, + description="Expected Kind for the children nodes in a Hierarchy, default to the main generic defined if not defined.", + ) + + +class GenericSchemaWrite(BaseNodeSchemaWrite): + model_config = ConfigDict(extra="ignore", use_enum_values=True) + hierarchical: bool = Field( + default=False, + description="Defines if the Generic support the hierarchical mode.", + ) + generate_profile: bool = Field( + default=True, + description="Indicate if a profile schema should be generated for this schema", + ) + restricted_namespaces: list[str] | None = Field( + default=None, + description="Nodes inheriting from this Generic schema must belong to one of the listed namespaces", + ) + + +class NodeExtensionWrite(BaseModel): + model_config = ConfigDict(extra="ignore", use_enum_values=True) + kind: str = Field( + ..., + description="Kind of the existing node to extend.", + ) + attributes: list[AttributeSchemaWrite] = Field( + default_factory=list, + description="Attributes to add to the existing node.", + ) + relationships: list[RelationshipSchemaWrite] = Field( + default_factory=list, + description="Relationships to add to the existing node.", + ) + + +class SchemaExtensionWrite(BaseModel): + model_config = ConfigDict(extra="ignore", use_enum_values=True) + nodes: list[NodeExtensionWrite] = Field( + default_factory=list, + description="Nodes to extend with additional attributes and relationships.", + ) + + +class InfrahubSchemaWrite(BaseModel): + model_config = ConfigDict(extra="ignore", use_enum_values=True) + version: str + nodes: list[NodeSchemaWrite] = Field(default_factory=list) + generics: list[GenericSchemaWrite] = Field(default_factory=list) + extensions: SchemaExtensionWrite | None = None diff --git a/infrahub_sdk/schema/main.py b/infrahub_sdk/schema/main.py index 83bc69a86..04352d965 100644 --- a/infrahub_sdk/schema/main.py +++ b/infrahub_sdk/schema/main.py @@ -1,155 +1,155 @@ from __future__ import annotations -import warnings from collections.abc import MutableMapping -from enum import Enum from typing import TYPE_CHECKING, Any from pydantic import BaseModel, ConfigDict, Field from typing_extensions import Self +from .generated.enums import ( + AllowOverrideType, + AttributeKind, + BranchSupportType, + ComputedAttributeKind, + RelationshipCardinality, + RelationshipDeleteBehavior, + RelationshipDirection, + RelationshipKind, + SchemaAttributeDisplay, + SchemaState, +) +from .generated.read import ( + AttributeSchemaBaseRead, + BaseNodeSchemaRead, + ComputedAttributeRead, # noqa: F401 (re-exported here to resolve the inherited forward reference) + GenericSchemaRead, + NodeSchemaRead, + ProfileSchemaRead, + RelationshipSchemaRead, + TemplateSchemaRead, +) +from .generated.write import ( + AttributeSchemaBaseWrite, + ComputedAttributeWrite, # noqa: F401 (re-exported here to resolve the inherited forward reference) + GenericSchemaWrite, + NodeSchemaWrite, + RelationshipSchemaWrite, + SchemaExtensionWrite, +) + if TYPE_CHECKING: from ..node import InfrahubNode, InfrahubNodeSync InfrahubNodeTypes = InfrahubNode | InfrahubNodeSync +# The enum classes and the generated write/read data models now live in the generated modules. +# ``main.py`` keeps the public names stable by re-exporting the enums and by subclassing the +# generated data models with the hand-written behavior below. The historical import paths +# (``from infrahub_sdk.schema.main import AttributeKind, NodeSchema, ...``) keep working. +__all__ = [ + "AllowOverrideType", + "AttributeKind", + "AttributeSchema", + "AttributeSchemaAPI", + "BranchSchema", + "BranchSupportType", + "ComputedAttributeKind", + "GenericSchema", + "GenericSchemaAPI", + "NodeSchema", + "NodeSchemaAPI", + "ProfileSchemaAPI", + "RelationshipCardinality", + "RelationshipDeleteBehavior", + "RelationshipDirection", + "RelationshipKind", + "RelationshipSchema", + "RelationshipSchemaAPI", + "SchemaAttributeDisplay", + "SchemaRoot", + "SchemaRootAPI", + "SchemaState", + "TemplateSchemaAPI", +] + + +# --------------------------------------------------------------------------- +# Write models (user-facing construction entry points) +# --------------------------------------------------------------------------- + + +class AttributeSchema(AttributeSchemaBaseWrite): + """Thin, constructible attribute model kept for backward compatibility. + + ``AttributeSchemaWrite`` (from the generated module) is a non-constructible discriminated union. + This class keeps ``AttributeSchema(name=..., kind=AttributeKind.TEXT, ...)`` working by exposing + the shared write base plus a permissive ``parameters``/``choices``. Unknown keys are dropped + silently (inherited ``extra="ignore"``), matching the rest of the write contract. + """ -class RelationshipCardinality(str, Enum): - ONE = "one" - MANY = "many" - - -class BranchSupportType(str, Enum): - AWARE = "aware" - AGNOSTIC = "agnostic" - LOCAL = "local" - - -class RelationshipKind(str, Enum): - GENERIC = "Generic" - ATTRIBUTE = "Attribute" - COMPONENT = "Component" - PARENT = "Parent" - GROUP = "Group" - HIERARCHY = "Hierarchy" - PROFILE = "Profile" - TEMPLATE = "Template" - - -class RelationshipDirection(str, Enum): - BIDIR = "bidirectional" - OUTBOUND = "outbound" - INBOUND = "inbound" - - -class AttributeKind(str, Enum): - ID = "ID" - TEXT = "Text" - STRING = "String" # deprecated - TEXTAREA = "TextArea" - DATETIME = "DateTime" - NUMBER = "Number" - NUMBERPOOL = "NumberPool" - DROPDOWN = "Dropdown" - EMAIL = "Email" - PASSWORD = "Password" # noqa: S105 - HASHEDPASSWORD = "HashedPassword" - URL = "URL" - FILE = "File" - MAC_ADDRESS = "MacAddress" - COLOR = "Color" - BANDWIDTH = "Bandwidth" - IPHOST = "IPHost" - IPNETWORK = "IPNetwork" - BOOLEAN = "Boolean" - CHECKBOX = "Checkbox" - LIST = "List" - JSON = "JSON" - ANY = "Any" - - def __getattr__(self, name: str) -> Any: - if name == "STRING": - warnings.warn( - f"{name} is deprecated and will be removed in future versions.", - DeprecationWarning, - stacklevel=2, - ) - return super().__getattribute__(name) - - -class SchemaState(str, Enum): - PRESENT = "present" - ABSENT = "absent" - - -class AllowOverrideType(str, Enum): - NONE = "none" - ANY = "any" - - -class RelationshipDeleteBehavior(str, Enum): - NO_ACTION = "no-action" - CASCADE = "cascade" - - -class AttributeSchema(BaseModel): - model_config = ConfigDict(use_enum_values=True) - - id: str | None = None - state: SchemaState = SchemaState.PRESENT - name: str - kind: AttributeKind - label: str | None = None - description: str | None = None - default_value: Any | None = None - unique: bool = False - branch: BranchSupportType | None = None - optional: bool = False choices: list[dict[str, Any]] | None = None - enum: list[str | int] | None = None - max_length: int | None = None - min_length: int | None = None - regex: str | None = None - order_weight: int | None = None + parameters: dict[str, Any] | None = None -class AttributeSchemaAPI(AttributeSchema): - model_config = ConfigDict(use_enum_values=True) +class RelationshipSchema(RelationshipSchemaWrite): + """Constructible relationship write model (kept as a distinct public name).""" + - inherited: bool = False - read_only: bool = False - allow_override: AllowOverrideType = AllowOverrideType.ANY +class NodeSchema(NodeSchemaWrite): + # The generated write model types these as discriminated unions, which cannot be instantiated + # directly. Overriding them with the public constructible models keeps + # ``NodeSchema(attributes=[AttributeSchema(...)])`` working for existing callers. + attributes: list[AttributeSchema] = Field(default_factory=list) + relationships: list[RelationshipSchema] = Field(default_factory=list) + + def convert_api(self) -> NodeSchemaAPI: + return NodeSchemaAPI(**self.model_dump()) -class RelationshipSchema(BaseModel): +class GenericSchema(GenericSchemaWrite): + attributes: list[AttributeSchema] = Field(default_factory=list) + relationships: list[RelationshipSchema] = Field(default_factory=list) + + def convert_api(self) -> GenericSchemaAPI: + return GenericSchemaAPI(**self.model_dump()) + + +class SchemaRoot(BaseModel): model_config = ConfigDict(use_enum_values=True) - id: str | None = None - state: SchemaState = SchemaState.PRESENT - name: str - peer: str - kind: RelationshipKind = RelationshipKind.GENERIC - label: str | None = None - description: str | None = None - identifier: str | None = None - min_count: int | None = None - max_count: int | None = None - direction: RelationshipDirection = RelationshipDirection.BIDIR - on_delete: RelationshipDeleteBehavior | None = None - cardinality: str = "many" - branch: BranchSupportType | None = None - optional: bool = True - order_weight: int | None = None - - -class RelationshipSchemaAPI(RelationshipSchema): + version: str + generics: list[GenericSchema] = Field(default_factory=list) + nodes: list[NodeSchema] = Field(default_factory=list) + # ``extensions`` mirrors the generated write contract (``nodes``/``generics``/``relationships`` + # under one block). It replaces the former flat ``node_extensions``, which the load endpoint no + # longer accepts. + extensions: SchemaExtensionWrite | None = None + + def to_schema_dict(self) -> dict[str, Any]: + return self.model_dump(exclude_unset=True, exclude_defaults=True) + + +# --------------------------------------------------------------------------- +# Read models (``*API``) -- concrete subclasses so ``isinstance`` keeps working +# --------------------------------------------------------------------------- + + +class AttributeSchemaAPI(AttributeSchemaBaseRead): + """Thin, constructible read-side attribute model kept for backward compatibility. + + ``AttributeSchemaRead`` (from the generated module) is a non-constructible discriminated union. + This class keeps ``AttributeSchemaAPI(name=..., kind=..., ...)`` working and is used as the item + type on the read schema models. It exposes the shared read base plus a permissive + ``parameters``/``choices``. No code performs ``isinstance`` on it. + """ + model_config = ConfigDict(use_enum_values=True) - inherited: bool = False - read_only: bool = False - hierarchical: str | None = None - allow_override: AllowOverrideType = AllowOverrideType.ANY + choices: list[dict[str, Any]] | None = None + parameters: dict[str, Any] | None = None + +class RelationshipSchemaAPI(RelationshipSchemaRead): @property def cardinality_is_one(self) -> bool: return self.cardinality == RelationshipCardinality.ONE @@ -159,12 +159,17 @@ def cardinality_is_many(self) -> bool: return self.cardinality == RelationshipCardinality.MANY -class BaseSchemaAttrRel(BaseModel): - attributes: list[AttributeSchema] = Field(default_factory=list) - relationships: list[RelationshipSchema] = Field(default_factory=list) +class _SchemaNodeBase(BaseNodeSchemaRead): + """Behavior shared by the node/generic/profile/template read models. + Subclasses ``BaseNodeSchemaRead``, so ``name``, ``namespace``, ``kind`` and the attribute/ + relationship collections are real inherited fields and the helpers below type-check against + them. The node-like ``*SchemaAPI`` classes inherit this alongside their specific read model + (diamond on ``BaseNodeSchemaRead``); listing this base first keeps the narrowed item types. + """ -class BaseSchemaAttrRelAPI(BaseModel): + # Narrow the attribute/relationship item types to the API variants so the returned items expose + # the behavior helpers (``cardinality_is_*``, ``inherited`` filtering, ...). attributes: list[AttributeSchemaAPI] = Field(default_factory=list) relationships: list[RelationshipSchemaAPI] = Field(default_factory=list) @@ -263,30 +268,6 @@ def local_relationships(self) -> list[RelationshipSchemaAPI]: def unique_attributes(self) -> list[AttributeSchemaAPI]: return [item for item in self.attributes if item.unique] - -class BaseSchema(BaseModel): - model_config = ConfigDict(use_enum_values=True) - - id: str | None = None - state: SchemaState = SchemaState.PRESENT - name: str - label: str | None = None - namespace: str - description: str | None = None - include_in_menu: bool | None = None - menu_placement: str | None = None - display_label: str | None = None - display_labels: list[str] | None = None - human_friendly_id: list[str] | None = None - icon: str | None = None - uniqueness_constraints: list[list[str]] | None = None - documentation: str | None = None - order_by: list[str] | None = None - - @property - def kind(self) -> str: - return self.namespace + self.name - @property def supports_artifact_definition(self) -> bool: """Returns True if this schema represents CoreArtifactDefinition. Only meaningful for NodeSchemaAPI.""" @@ -324,41 +305,7 @@ def hierarchical_relationship_schemas(self) -> list[RelationshipSchemaAPI]: return [] -class GenericSchema(BaseSchema, BaseSchemaAttrRel): - def convert_api(self) -> GenericSchemaAPI: - return GenericSchemaAPI(**self.model_dump()) - - -class GenericSchemaAPI(BaseSchema, BaseSchemaAttrRelAPI): - """A Generic can be either an Interface or a Union depending if there are some Attributes or Relationships defined.""" - - hash: str | None = None - hierarchical: bool | None = None - used_by: list[str] = Field(default_factory=list) - restricted_namespaces: list[str] | None = None - - -class BaseNodeSchema(BaseSchema): - model_config = ConfigDict(use_enum_values=True) - - inherit_from: list[str] = Field(default_factory=list) - branch: BranchSupportType | None = None - default_filter: str | None = None - generate_profile: bool | None = None - generate_template: bool | None = None - parent: str | None = None - children: str | None = None - - -class NodeSchema(BaseNodeSchema, BaseSchemaAttrRel): - def convert_api(self) -> NodeSchemaAPI: - return NodeSchemaAPI(**self.model_dump()) - - -class NodeSchemaAPI(BaseNodeSchema, BaseSchemaAttrRelAPI): - hash: str | None = None - hierarchy: str | None = None - +class NodeSchemaAPI(_SchemaNodeBase, NodeSchemaRead): @property def supports_artifacts(self) -> bool: return "CoreArtifactTarget" in self.inherit_from @@ -377,52 +324,46 @@ def hierarchical_relationship_schemas(self) -> list[RelationshipSchemaAPI]: return [] return [ RelationshipSchemaAPI( - name="parent", peer=self.hierarchy, kind=RelationshipKind.HIERARCHY, cardinality="one", optional=True + name="parent", + peer=self.hierarchy, + kind=RelationshipKind.HIERARCHY, + cardinality=RelationshipCardinality.ONE, + optional=True, ), RelationshipSchemaAPI( - name="children", peer=self.hierarchy, kind=RelationshipKind.HIERARCHY, cardinality="many", optional=True + name="children", + peer=self.hierarchy, + kind=RelationshipKind.HIERARCHY, + cardinality=RelationshipCardinality.MANY, + optional=True, ), RelationshipSchemaAPI( - name="ancestors", peer=self.hierarchy, cardinality="many", read_only=True, optional=True + name="ancestors", + peer=self.hierarchy, + cardinality=RelationshipCardinality.MANY, + read_only=True, + optional=True, ), RelationshipSchemaAPI( - name="descendants", peer=self.hierarchy, cardinality="many", read_only=True, optional=True + name="descendants", + peer=self.hierarchy, + cardinality=RelationshipCardinality.MANY, + read_only=True, + optional=True, ), ] -class ProfileSchemaAPI(BaseSchema, BaseSchemaAttrRelAPI): - inherit_from: list[str] = Field(default_factory=list) - - -class TemplateSchemaAPI(BaseSchema, BaseSchemaAttrRelAPI): - inherit_from: list[str] = Field(default_factory=list) - - -class NodeExtensionSchema(BaseModel): - model_config = ConfigDict(use_enum_values=True) - - name: str | None = None - kind: str - description: str | None = None - label: str | None = None - inherit_from: list[str] = Field(default_factory=list) - branch: BranchSupportType | None = None - default_filter: str | None = None - attributes: list[AttributeSchema] = Field(default_factory=list) - relationships: list[RelationshipSchema] = Field(default_factory=list) +class GenericSchemaAPI(_SchemaNodeBase, GenericSchemaRead): + """A Generic can be either an Interface or a Union depending if there are some Attributes or Relationships defined.""" -class SchemaRoot(BaseModel): - model_config = ConfigDict(use_enum_values=True) +class ProfileSchemaAPI(_SchemaNodeBase, ProfileSchemaRead): + pass - version: str - generics: list[GenericSchema] = Field(default_factory=list) - nodes: list[NodeSchema] = Field(default_factory=list) - node_extensions: list[NodeExtensionSchema] = Field(default_factory=list) - def to_schema_dict(self) -> dict[str, Any]: - return self.model_dump(exclude_unset=True, exclude_defaults=True) +class TemplateSchemaAPI(_SchemaNodeBase, TemplateSchemaRead): + pass class SchemaRootAPI(BaseModel): diff --git a/infrahub_sdk/schema/repository.py b/infrahub_sdk/schema/repository.py index aca13a78b..db3bece9f 100644 --- a/infrahub_sdk/schema/repository.py +++ b/infrahub_sdk/schema/repository.py @@ -37,6 +37,31 @@ class InfrahubRepositoryArtifactDefinitionConfig(InfrahubRepositoryConfigElement transformation: str = Field(..., description="The transformation to use.") +MISSING_WATCH_MESSAGE = ( + "Missing 'watch' block. Infrahub cannot detect every file this depends on, so list templates, " + "helper modules and data files under 'watch.files' to have the results regenerated when they " + "change. Use an empty 'files: []' to record that nothing extra needs watching." +) + +MALFORMED_WATCH_MESSAGE = ( + "The 'watch' block must be a mapping. Put the watched paths under 'files', or use 'files: []' to " + "confirm that nothing extra needs watching." +) + +# Advisory only: nothing here is enforced by the models. A YAML language server reports a missing +# required property as a warning, which is how editors flag an absent 'watch' block while someone +# edits .infrahub.yml. 'allOf' is used rather than a top-level 'required' because keys in +# json_schema_extra replace the ones pydantic generates, which would drop the real required fields. +REQUIRE_WATCH_JSON_SCHEMA: dict[str, Any] = {"allOf": [{"required": ["watch"], "errorMessage": MISSING_WATCH_MESSAGE}]} + +# Narrowing the field to an object rejects a bare 'watch:' or 'watch: null', which pydantic accepts +# through the null half of the generated anyOf. Those forms leave 'watch' as None, indistinguishable +# from never having declared it, so the acknowledgement the block exists to record is lost. An empty +# 'watch: {}' is deliberately fine: 'files' defaults to an empty list, so it carries that +# acknowledgement without the author having to spell the key out. +WATCH_FIELD_JSON_SCHEMA: dict[str, Any] = {"type": "object", "errorMessage": MALFORMED_WATCH_MESSAGE} + + class InfrahubWatchConfig(BaseModel): """Extra files and directories a transform depends on. @@ -44,6 +69,9 @@ class InfrahubWatchConfig(BaseModel): depends on files that cannot be detected automatically, such as templates pulled in dynamically or helper modules imported at runtime. When any watched file changes, the transform's artifacts are regenerated. + + An empty 'files' list is a valid answer: it records that the author checked and nothing beyond + what Infrahub detects needs watching. """ model_config = ConfigDict(extra="forbid") @@ -61,6 +89,7 @@ class InfrahubJinja2TransformConfig(InfrahubRepositoryConfigElement): description: str | None = Field(default=None, description="Description for this transform") watch: InfrahubWatchConfig | None = Field( default=None, + json_schema_extra=WATCH_FIELD_JSON_SCHEMA, description="Extra files and directories this transform depends on, in addition to the ones Infrahub detects automatically.", ) @@ -102,7 +131,7 @@ def load_class(self, import_root: str | None = None, relative_path: str | None = class InfrahubGeneratorDefinitionConfig(InfrahubRepositoryConfigElement): - model_config = ConfigDict(extra="forbid") + model_config = ConfigDict(extra="forbid", json_schema_extra=REQUIRE_WATCH_JSON_SCHEMA) name: str = Field(..., description="The name of the Generator Definition") file_path: Path = Field(..., description="The file within the repository with the generator code.") @@ -131,6 +160,11 @@ class InfrahubGeneratorDefinitionConfig(InfrahubRepositoryConfigElement): default=True, description="When true (default), the Generator runs after a branch merge. Set to false for Generators that only run via event triggers.", ) + watch: InfrahubWatchConfig | None = Field( + default=None, + json_schema_extra=WATCH_FIELD_JSON_SCHEMA, + description="Extra files and directories this generator depends on, in addition to the ones Infrahub detects automatically.", + ) def load_class(self, import_root: str | None = None, relative_path: str | None = None) -> type[InfrahubGenerator]: module = import_module(module_path=self.file_path, import_root=import_root, relative_path=relative_path) @@ -147,7 +181,7 @@ def load_class(self, import_root: str | None = None, relative_path: str | None = class InfrahubPythonTransformConfig(InfrahubRepositoryConfigElement): - model_config = ConfigDict(extra="forbid") + model_config = ConfigDict(extra="forbid", json_schema_extra=REQUIRE_WATCH_JSON_SCHEMA) name: str = Field(..., description="The name of the Transform") file_path: Path = Field(..., description="The file within the repository with the transform code.") @@ -159,6 +193,7 @@ class InfrahubPythonTransformConfig(InfrahubRepositoryConfigElement): description: str | None = Field(default=None, description="Description for this transform") watch: InfrahubWatchConfig | None = Field( default=None, + json_schema_extra=WATCH_FIELD_JSON_SCHEMA, description="Extra files and directories this transform depends on, in addition to the ones Infrahub detects automatically.", ) diff --git a/infrahub_sdk/schema/validate.py b/infrahub_sdk/schema/validate.py new file mode 100644 index 000000000..8cae66cb1 --- /dev/null +++ b/infrahub_sdk/schema/validate.py @@ -0,0 +1,260 @@ +"""Offline validation of a schema payload against the generated write models. + +This module depends only on pydantic and the generated write models, so a caller +can validate a schema payload with just the SDK installed (no server, no backend). +The write models omit fields the user may not set (read-level, internal) and set +``extra="ignore"``, so those values never reach the server. Whether an omitted field is +reported depends on what it is: a read-only field -- one the read API returns -- is reported +as a warning so a payload read back from Infrahub still loads, while any other extra field is +an error, because the only ways to get one are a typo and a field that no longer exists. +Constrained fields set outside their allowed set are also rejected naming the field and the +invalid value, as are missing required fields and unknown enum members. +""" + +from __future__ import annotations + +from typing import Any + +from pydantic import BaseModel, Field +from pydantic import ValidationError as PydanticValidationError + +from .generated.contract import READ_ONLY_FIELDS +from .generated.write import InfrahubSchemaWrite + +# Payload containers whose items carry the identity used to report a finding. The warning shape +# consumers render is named (kind, field) rather than positional, so the walk tracks the owning +# kind and element alongside the dotted path. +_KIND_CONTAINERS = frozenset({"nodes", "generics"}) +_ELEMENT_CONTAINERS = frozenset({"attributes", "relationships"}) + + +class SchemaValidationErrorDetail(BaseModel): + """A single field-level validation problem in a schema payload.""" + + field: str = Field(..., description="Dotted path to the offending field, e.g. 'nodes[0].attributes[1].kind'") + message: str = Field(..., description="Human-readable, field-level error message") + + +class SchemaValidationWarningDetail(BaseModel): + """A read-only field set in a schema payload: accepted, but the submitted value is dropped.""" + + field: str = Field(..., description="Dotted path to the offending field, e.g. 'nodes[0].attributes[1].inherited'") + name: str = Field( + ..., + description="Field path relative to the owning kind or element, e.g. 'inherited' or 'parameters.id'", + ) + kind: str | None = Field(default=None, description="Kind of the schema node carrying the field, when resolvable") + element: str | None = Field( + default=None, description="Name of the attribute or relationship carrying the field, when applicable" + ) + message: str = Field(..., description="Human-readable, field-level warning message") + + +class SchemaValidationResult(BaseModel): + """The verdict of validating a schema payload against the write contract.""" + + valid: bool = Field(..., description="True when the payload satisfies the write contract") + errors: list[SchemaValidationErrorDetail] = Field( + default_factory=list, description="One entry per field-level problem; empty when valid" + ) + warnings: list[SchemaValidationWarningDetail] = Field( + default_factory=list, description="One entry per read-only field set in the payload" + ) + + @property + def messages(self) -> list[str]: + return [error.message for error in self.errors] + + @property + def warning_messages(self) -> list[str]: + return [warning.message for warning in self.warnings] + + def raise_for_status(self) -> None: + """Raise when the payload is invalid, joining every field-level message. + + Raises: + ValueError: When the result is invalid; the message joins every field-level error. + + """ + if not self.valid: + raise ValueError("; ".join(self.messages)) + + +def _format_error_location(loc: tuple[Any, ...], prefix: str = "") -> str: + """Render a dotted field path from a pydantic error location, optionally under a base prefix. + + Integer elements index into the preceding segment (``attributes`` + ``1`` becomes + ``attributes[1]``); everything else is appended as a new dotted segment. A ``prefix`` is used + when the location is relative to an item validated on its own (e.g. an extension attribute). + """ + parts = [prefix] if prefix else [] + for element in loc: + if isinstance(element, int): + if parts: + parts[-1] = f"{parts[-1]}[{element}]" + else: + parts.append(f"[{element}]") + else: + parts.append(str(element)) + return ".".join(parts) + + +def _collect_validation_errors( + exc: PydanticValidationError, errors: list[SchemaValidationErrorDetail], prefix: str = "" +) -> None: + """Append a field-level detail for every problem in a pydantic validation error.""" + for error in exc.errors(): + location = _format_error_location(loc=error["loc"], prefix=prefix) + message = f"{location}: {error['msg']}" + if error["type"] != "missing" and "input" in error: + message += f" (received: {error['input']!r})" + errors.append(SchemaValidationErrorDetail(field=location, message=message)) + + +def _descend_context( + field: str, item: dict[str, Any], kind: str | None, element: str | None, qualifier: tuple[str, ...] +) -> tuple[str | None, str | None, tuple[str, ...]]: + """Resolve the owning kind, element and field qualifier for a value nested under a field. + + Entering a kind or element container re-anchors the identity a finding is reported against, so + the qualifier resets there. Anywhere else the field name joins the qualifier, which is what + distinguishes a nested ``parameters.id`` from an ``id`` set directly on the attribute. + """ + if field in _KIND_CONTAINERS: + namespace, name = item.get("namespace"), item.get("name") + # An extension addresses an existing node by kind; a new node is namespace + name. + resolved = f"{namespace}{name}" if namespace and name else item.get("kind") + return (resolved if isinstance(resolved, str) else None), None, () + if field in _ELEMENT_CONTAINERS: + name = item.get("name") + return kind, (name if isinstance(name, str) else None), () + return kind, element, (*qualifier, field) + + +def _collect_extra_fields( + payload: Any, + instance: BaseModel, + errors: list[SchemaValidationErrorDetail], + warnings: list[SchemaValidationWarningDetail], + path: str = "", + field: str | None = None, + kind: str | None = None, + element: str | None = None, + qualifier: tuple[str, ...] = (), +) -> None: + """Report every payload key the write contract does not declare, walking the validated model. + + The submitted payload is walked alongside the model validated from it, because neither on its + own carries what a finding needs. ``extra="ignore"`` means the validated instance no longer + knows which keys were dropped, so the raw payload has to supply them; and only the instance + resolves which model governs a given location -- notably which member of a discriminated union + an attribute matched -- so only it can say which keys that location accepts. Pairing them also + yields the owning kind and element, which a finding is reported against by name rather than by + position. + + ``field`` is the name this payload was reached through, and is None only at the root. + """ + if not isinstance(payload, dict): + # A caller may nest an already-built model rather than plain data; a model cannot carry an + # undeclared key, so there is nothing to compare and nothing below worth walking. + return + + if field is not None: + kind, element, qualifier = _descend_context( + field=field, item=payload, kind=kind, element=element, qualifier=qualifier + ) + + fields = type(instance).model_fields + read_only = READ_ONLY_FIELDS.get(type(instance).__name__, frozenset()) + + for key in sorted(set(payload) - set(fields)): + location = f"{path}.{key}" if path else key + if key in read_only: + warnings.append( + SchemaValidationWarningDetail( + field=location, + name=".".join((*qualifier, key)), + kind=kind, + element=element, + message=f"{location}: Read-only field, the submitted value is ignored (received: {payload[key]!r})", + ) + ) + else: + errors.append( + SchemaValidationErrorDetail( + field=location, + message=f"{location}: Unknown field, it is not part of the schema (received: {payload[key]!r})", + ) + ) + + for name in fields: + if name not in payload: + continue + raw, value = payload[name], getattr(instance, name) + child_path = f"{path}.{name}" if path else name + # Validation succeeded, so a list field is index-aligned with the list it was built from. + # A list of plain values carries no nested model and is skipped. + if isinstance(value, list): + for index, (raw_item, item) in enumerate(zip(raw, value, strict=True)): + if isinstance(item, BaseModel): + _collect_extra_fields( + payload=raw_item, + instance=item, + errors=errors, + warnings=warnings, + path=f"{child_path}[{index}]", + field=name, + kind=kind, + element=element, + qualifier=qualifier, + ) + elif isinstance(value, BaseModel): + _collect_extra_fields( + payload=raw, + instance=value, + errors=errors, + warnings=warnings, + path=child_path, + field=name, + kind=kind, + element=element, + qualifier=qualifier, + ) + + +def validate_schema(schema: dict[str, Any], *, raise_on_error: bool = False) -> SchemaValidationResult: + """Validate a single schema-root payload against the generated write contract. + + Args: + schema: A schema-root mapping, e.g. ``{"version": "1.0", "nodes": [...], "generics": [...]}``. + raise_on_error: When True, raise ``ValueError`` instead of returning an invalid result. + + Returns: + A :class:`SchemaValidationResult` with a field-level message for every constrained field set + outside its allowed set, every missing required field, and every extra field the contract + does not declare, plus a warning for every read-only field the payload sets. The whole root + -- nodes, generics and the attributes/relationships nested under ``extensions.nodes`` -- is + validated against the write document model in one pass. + + Extra fields are reported only once the payload validates against the write models, since + the validated instance is what resolves the contract applying at each location. A payload + rejected for another reason therefore reports that reason first. + + Raises: + ValueError: When ``raise_on_error`` is True and the payload is invalid. + + """ + errors: list[SchemaValidationErrorDetail] = [] + warnings: list[SchemaValidationWarningDetail] = [] + + try: + validated = InfrahubSchemaWrite.model_validate(schema) + except PydanticValidationError as exc: + _collect_validation_errors(exc=exc, errors=errors) + else: + _collect_extra_fields(payload=schema, instance=validated, errors=errors, warnings=warnings) + + result = SchemaValidationResult(valid=not errors, errors=errors, warnings=warnings) + if raise_on_error: + result.raise_for_status() + return result diff --git a/infrahub_sdk/spec/object.py b/infrahub_sdk/spec/object.py index d634442a7..432d37d12 100644 --- a/infrahub_sdk/spec/object.py +++ b/infrahub_sdk/spec/object.py @@ -7,7 +7,7 @@ from pydantic import BaseModel, Field from ..exceptions import ObjectValidationError, ValidationError -from ..schema import GenericSchemaAPI, RelationshipCardinality, RelationshipKind, RelationshipSchema +from ..schema import GenericSchemaAPI, RelationshipCardinality, RelationshipKind, RelationshipSchemaAPI from ..utils import is_valid_uuid from ..yaml import InfrahubFile, InfrahubFileKind from .models import InfrahubObjectParameters @@ -16,7 +16,7 @@ if TYPE_CHECKING: from ..client import InfrahubClient from ..node import InfrahubNode - from ..schema import MainSchemaTypesAPI, RelationshipSchema + from ..schema import MainSchemaTypesAPI def validate_list_of_scalars(value: list[Any]) -> bool: @@ -79,9 +79,9 @@ class RelationshipDataFormat(str, Enum): class RelationshipInfo(BaseModel): name: str - rel_schema: RelationshipSchema + rel_schema: RelationshipSchemaAPI peer_kind: str - peer_rel: RelationshipSchema | None = None + peer_rel: RelationshipSchemaAPI | None = None reason_relationship_not_valid: str | None = None format: RelationshipDataFormat = RelationshipDataFormat.UNKNOWN peer_human_friendly_id: list[str] | None = None @@ -128,7 +128,7 @@ def get_context(self, value: Any) -> dict: def find_matching_relationship( self, peer_schema: MainSchemaTypesAPI, force: bool = False - ) -> RelationshipSchema | None: + ) -> RelationshipSchemaAPI | None: """Find the matching relationship on the other side of the relationship.""" if self.peer_rel and not force: return self.peer_rel diff --git a/infrahub_sdk/task/__init__.py b/infrahub_sdk/task/__init__.py index 601803158..7ad13ccc5 100644 --- a/infrahub_sdk/task/__init__.py +++ b/infrahub_sdk/task/__init__.py @@ -1,11 +1,29 @@ from __future__ import annotations -from .models import Task, TaskFilter, TaskLog, TaskRelatedNode, TaskState +from .models import ( + HttpRequest, + HttpResponse, + Task, + TaskAction, + TaskActionName, + TaskError, + TaskFilter, + TaskLog, + TaskRelatedNode, + TaskState, + WebhookDeliveryTask, +) __all__ = [ + "HttpRequest", + "HttpResponse", "Task", + "TaskAction", + "TaskActionName", + "TaskError", "TaskFilter", "TaskLog", "TaskRelatedNode", "TaskState", + "WebhookDeliveryTask", ] diff --git a/infrahub_sdk/task/manager.py b/infrahub_sdk/task/manager.py index 913c6b753..59f17f869 100644 --- a/infrahub_sdk/task/manager.py +++ b/infrahub_sdk/task/manager.py @@ -4,7 +4,7 @@ import time from typing import TYPE_CHECKING, Any -from ..graphql import Query +from ..graphql import Mutation, Query from .constants import FINAL_STATES from .exceptions import TaskNotCompletedError, TaskNotFoundError, TooManyTasksError from .models import Task, TaskFilter @@ -12,6 +12,8 @@ if TYPE_CHECKING: from ..client import InfrahubClient, InfrahubClientSync +MUTATION_TASK_QUERY = {"ok": None, "task": {"id": None}} + class InfraHubTaskManagerBase: @classmethod @@ -20,6 +22,8 @@ def _generate_query( filters: TaskFilter | None = None, include_logs: bool = False, include_related_nodes: bool = False, + include_actions: bool = False, + include_diagnostics: bool = False, offset: int | None = None, limit: int | None = None, count: bool = False, @@ -68,6 +72,22 @@ def _generate_query( if include_related_nodes: query["InfrahubTask"]["edges"]["node"]["related_nodes"] = {"id": None, "kind": None} + if include_actions: + query["InfrahubTask"]["edges"]["node"]["available_actions"] = { + "action": None, + "available": None, + "unavailability_reason": None, + } + + if include_diagnostics: + node = query["InfrahubTask"]["edges"]["node"] + node["error"] = {"status_class": None, "message": None, "remediation": None} + # node is a GraphQL interface; the string key renders as an inline fragment. + node["... on WebhookDeliveryTask"] = { + "http_request": {"url": None, "headers": None}, + "http_response": {"status_code": None, "body": None, "latency_ms": None}, + } + return Query(query=query) @classmethod @@ -113,6 +133,8 @@ async def all( parallel: bool = False, include_logs: bool = False, include_related_nodes: bool = False, + include_actions: bool = False, + include_diagnostics: bool = False, ) -> list[Task]: """Get all tasks. @@ -123,6 +145,8 @@ async def all( parallel: Whether to query the tasks in parallel. Defaults to False. include_logs: Whether to include the logs in the tasks. Defaults to False. include_related_nodes: Whether to include the related nodes in the tasks. Defaults to False. + include_actions: Whether to include the available actions in the tasks. Defaults to False. + include_diagnostics: Whether to include the error and webhook delivery diagnostics in the tasks. Defaults to False. Returns: A list of tasks. @@ -135,6 +159,8 @@ async def all( parallel=parallel, include_logs=include_logs, include_related_nodes=include_related_nodes, + include_actions=include_actions, + include_diagnostics=include_diagnostics, ) async def filter( @@ -146,6 +172,8 @@ async def filter( parallel: bool = False, include_logs: bool = False, include_related_nodes: bool = False, + include_actions: bool = False, + include_diagnostics: bool = False, ) -> list[Task]: """Filter tasks. @@ -157,6 +185,8 @@ async def filter( parallel: Whether to query the tasks in parallel. Defaults to False. include_logs: Whether to include the logs in the tasks. Defaults to False. include_related_nodes: Whether to include the related nodes in the tasks. Defaults to False. + include_actions: Whether to include the available actions in the tasks. Defaults to False. + include_diagnostics: Whether to include the error and webhook delivery diagnostics in the tasks. Defaults to False. Returns: A list of tasks. @@ -167,7 +197,19 @@ async def filter( if limit: tasks, _ = await self.process_page( - self.client, self._generate_query(filters=filter, offset=offset, limit=limit, count=False), 1, timeout + self.client, + self._generate_query( + filters=filter, + offset=offset, + limit=limit, + include_logs=include_logs, + include_related_nodes=include_related_nodes, + include_actions=include_actions, + include_diagnostics=include_diagnostics, + count=False, + ), + 1, + timeout, ) return tasks @@ -177,6 +219,8 @@ async def filter( timeout=timeout, include_logs=include_logs, include_related_nodes=include_related_nodes, + include_actions=include_actions, + include_diagnostics=include_diagnostics, ) return await self.process_non_batch( @@ -186,13 +230,24 @@ async def filter( timeout=timeout, include_logs=include_logs, include_related_nodes=include_related_nodes, + include_actions=include_actions, + include_diagnostics=include_diagnostics, ) - async def get(self, id: str, include_logs: bool = False, include_related_nodes: bool = False) -> Task: + async def get( + self, + id: str, + include_logs: bool = False, + include_related_nodes: bool = False, + include_actions: bool = False, + include_diagnostics: bool = False, + ) -> Task: tasks = await self.filter( filter=TaskFilter(ids=[id]), include_logs=include_logs, include_related_nodes=include_related_nodes, + include_actions=include_actions, + include_diagnostics=include_diagnostics, parallel=False, ) if not tasks: @@ -225,6 +280,34 @@ async def wait_for_completion(self, id: str, interval: int = 1, timeout: int = 6 await asyncio.sleep(interval) raise TaskNotCompletedError(id=id, message=f"Task {id} did not complete in {timeout} seconds") + async def retry(self, id: str) -> str: + """Retry a settled task by replaying it as a new, independent task. + + Args: + id: The id of the task to retry. + + Returns: + The id of the new task created by the retry. + + """ + query = Mutation(mutation="InfrahubTaskRetry", input_data={"data": {"id": id}}, query=MUTATION_TASK_QUERY) + response = await self.client.execute_graphql(query=query.render(), tracker="mutation-task-retry") + return response["InfrahubTaskRetry"]["task"]["id"] + + async def cancel(self, id: str) -> bool: + """Cancel an in-flight task, stopping any remaining retries. + + Args: + id: The id of the task to cancel. + + Returns: + Whether the task was successfully cancelled. + + """ + query = Mutation(mutation="InfrahubTaskCancel", input_data={"data": {"id": id}}, query=MUTATION_TASK_QUERY) + response = await self.client.execute_graphql(query=query.render(), tracker="mutation-task-cancel") + return response["InfrahubTaskCancel"]["ok"] + @staticmethod async def process_page( client: InfrahubClient, query: Query, page_number: int, timeout: int | None = None @@ -255,6 +338,8 @@ async def process_batch( timeout: int | None = None, include_logs: bool = False, include_related_nodes: bool = False, + include_actions: bool = False, + include_diagnostics: bool = False, ) -> list[Task]: """Process queries in parallel mode.""" pagination_size = self.client.pagination_size @@ -271,6 +356,8 @@ async def process_batch( limit=pagination_size, include_logs=include_logs, include_related_nodes=include_related_nodes, + include_actions=include_actions, + include_diagnostics=include_diagnostics, count=False, ) batch_process.add( @@ -290,6 +377,8 @@ async def process_non_batch( timeout: int | None = None, include_logs: bool = False, include_related_nodes: bool = False, + include_actions: bool = False, + include_diagnostics: bool = False, ) -> list[Task]: """Process queries without parallel mode. @@ -309,6 +398,8 @@ async def process_non_batch( limit=self.client.pagination_size, include_logs=include_logs, include_related_nodes=include_related_nodes, + include_actions=include_actions, + include_diagnostics=include_diagnostics, count=True, ) new_tasks, count = await self.process_page( @@ -353,6 +444,8 @@ def all( parallel: bool = False, include_logs: bool = False, include_related_nodes: bool = False, + include_actions: bool = False, + include_diagnostics: bool = False, ) -> list[Task]: """Get all tasks. @@ -363,6 +456,8 @@ def all( parallel: Whether to query the tasks in parallel. Defaults to False. include_logs: Whether to include the logs in the tasks. Defaults to False. include_related_nodes: Whether to include the related nodes in the tasks. Defaults to False. + include_actions: Whether to include the available actions in the tasks. Defaults to False. + include_diagnostics: Whether to include the error and webhook delivery diagnostics in the tasks. Defaults to False. Returns: A list of tasks. @@ -375,6 +470,8 @@ def all( parallel=parallel, include_logs=include_logs, include_related_nodes=include_related_nodes, + include_actions=include_actions, + include_diagnostics=include_diagnostics, ) def filter( @@ -386,6 +483,8 @@ def filter( parallel: bool = False, include_logs: bool = False, include_related_nodes: bool = False, + include_actions: bool = False, + include_diagnostics: bool = False, ) -> list[Task]: """Filter tasks. @@ -397,6 +496,8 @@ def filter( parallel: Whether to query the tasks in parallel. Defaults to False. include_logs: Whether to include the logs in the tasks. Defaults to False. include_related_nodes: Whether to include the related nodes in the tasks. Defaults to False. + include_actions: Whether to include the available actions in the tasks. Defaults to False. + include_diagnostics: Whether to include the error and webhook delivery diagnostics in the tasks. Defaults to False. Returns: A list of tasks. @@ -407,7 +508,19 @@ def filter( if limit: tasks, _ = self.process_page( - self.client, self._generate_query(filters=filter, offset=offset, limit=limit, count=False), 1, timeout + self.client, + self._generate_query( + filters=filter, + offset=offset, + limit=limit, + include_logs=include_logs, + include_related_nodes=include_related_nodes, + include_actions=include_actions, + include_diagnostics=include_diagnostics, + count=False, + ), + 1, + timeout, ) return tasks @@ -417,6 +530,8 @@ def filter( timeout=timeout, include_logs=include_logs, include_related_nodes=include_related_nodes, + include_actions=include_actions, + include_diagnostics=include_diagnostics, ) return self.process_non_batch( @@ -426,13 +541,24 @@ def filter( timeout=timeout, include_logs=include_logs, include_related_nodes=include_related_nodes, + include_actions=include_actions, + include_diagnostics=include_diagnostics, ) - def get(self, id: str, include_logs: bool = False, include_related_nodes: bool = False) -> Task: + def get( + self, + id: str, + include_logs: bool = False, + include_related_nodes: bool = False, + include_actions: bool = False, + include_diagnostics: bool = False, + ) -> Task: tasks = self.filter( filter=TaskFilter(ids=[id]), include_logs=include_logs, include_related_nodes=include_related_nodes, + include_actions=include_actions, + include_diagnostics=include_diagnostics, parallel=False, ) if not tasks: @@ -465,6 +591,34 @@ def wait_for_completion(self, id: str, interval: int = 1, timeout: int = 60) -> time.sleep(interval) raise TaskNotCompletedError(id=id, message=f"Task {id} did not complete in {timeout} seconds") + def retry(self, id: str) -> str: + """Retry a settled task by replaying it as a new, independent task. + + Args: + id: The id of the task to retry. + + Returns: + The id of the new task created by the retry. + + """ + query = Mutation(mutation="InfrahubTaskRetry", input_data={"data": {"id": id}}, query=MUTATION_TASK_QUERY) + response = self.client.execute_graphql(query=query.render(), tracker="mutation-task-retry") + return response["InfrahubTaskRetry"]["task"]["id"] + + def cancel(self, id: str) -> bool: + """Cancel an in-flight task, stopping any remaining retries. + + Args: + id: The id of the task to cancel. + + Returns: + Whether the task was successfully cancelled. + + """ + query = Mutation(mutation="InfrahubTaskCancel", input_data={"data": {"id": id}}, query=MUTATION_TASK_QUERY) + response = self.client.execute_graphql(query=query.render(), tracker="mutation-task-cancel") + return response["InfrahubTaskCancel"]["ok"] + @staticmethod def process_page( client: InfrahubClientSync, query: Query, page_number: int, timeout: int | None = None @@ -495,6 +649,8 @@ def process_batch( timeout: int | None = None, include_logs: bool = False, include_related_nodes: bool = False, + include_actions: bool = False, + include_diagnostics: bool = False, ) -> list[Task]: """Process queries in parallel mode.""" pagination_size = self.client.pagination_size @@ -511,6 +667,8 @@ def process_batch( limit=pagination_size, include_logs=include_logs, include_related_nodes=include_related_nodes, + include_actions=include_actions, + include_diagnostics=include_diagnostics, count=False, ) batch_process.add( @@ -530,6 +688,8 @@ def process_non_batch( timeout: int | None = None, include_logs: bool = False, include_related_nodes: bool = False, + include_actions: bool = False, + include_diagnostics: bool = False, ) -> list[Task]: """Process queries without parallel mode. @@ -549,6 +709,8 @@ def process_non_batch( limit=self.client.pagination_size, include_logs=include_logs, include_related_nodes=include_related_nodes, + include_actions=include_actions, + include_diagnostics=include_diagnostics, count=True, ) new_tasks, count = self.process_page( diff --git a/infrahub_sdk/task/models.py b/infrahub_sdk/task/models.py index 2525bda21..9ab907b13 100644 --- a/infrahub_sdk/task/models.py +++ b/infrahub_sdk/task/models.py @@ -2,9 +2,12 @@ from datetime import datetime from enum import Enum +from typing import Any from pydantic import BaseModel, Field +WEBHOOK_SEND_WORKFLOW = "webhook-send" + class TaskState(str, Enum): SCHEDULED = "SCHEDULED" @@ -18,17 +21,47 @@ class TaskState(str, Enum): CANCELLING = "CANCELLING" +class TaskActionName(str, Enum): + RETRY = "RETRY" + CANCEL = "CANCEL" + + class TaskLog(BaseModel): message: str severity: str timestamp: datetime +class TaskAction(BaseModel): + action: TaskActionName + available: bool + unavailability_reason: str | None = None + + class TaskRelatedNode(BaseModel): id: str kind: str +class TaskError(BaseModel): + """Classified failure reason with a remediation hint; set only when a task failed with one.""" + + status_class: str + message: str + remediation: str | None = None + + +class HttpRequest(BaseModel): + url: str + headers: dict[str, Any] | None = None # secret values are masked by the server + + +class HttpResponse(BaseModel): + status_code: int | None = None + body: str | None = None + latency_ms: float | None = None + + class Task(BaseModel): id: str title: str @@ -43,11 +76,25 @@ class Task(BaseModel): tags: list[str] | None = None related_nodes: list[TaskRelatedNode] = Field(default_factory=list) logs: list[TaskLog] = Field(default_factory=list) + available_actions: list[TaskAction] = Field(default_factory=list) + error: TaskError | None = None + + @property + def can_retry(self) -> bool: + """Whether this task can currently be retried.""" + return any(action.action is TaskActionName.RETRY and action.available for action in self.available_actions) + + @property + def can_cancel(self) -> bool: + """Whether this task can currently be cancelled.""" + return any(action.action is TaskActionName.CANCEL and action.available for action in self.available_actions) @classmethod def from_graphql(cls, data: dict) -> Task: + data = dict(data) related_nodes: list[TaskRelatedNode] = [] logs: list[TaskLog] = [] + available_actions: list[TaskAction] = [] if "related_nodes" in data: if data.get("related_nodes"): @@ -59,7 +106,32 @@ def from_graphql(cls, data: dict) -> Task: logs = [TaskLog(**item["node"]) for item in data["logs"]["edges"]] del data["logs"] - return cls(**data, related_nodes=related_nodes, logs=logs) + if "available_actions" in data: + if data.get("available_actions"): + available_actions = [TaskAction(**item) for item in data["available_actions"]] + del data["available_actions"] + + # The workflow name selects the concrete type, mirroring the server's interface; + # pydantic coerces the remaining error / http_* dicts in `data` into their models. + target_cls = TASK_TYPES.get(data.get("workflow") or "", cls) + return target_cls( + **data, + related_nodes=related_nodes, + logs=logs, + available_actions=available_actions, + ) + + +class WebhookDeliveryTask(Task): + """Concrete task type for the ``webhook-send`` workflow, carrying delivery diagnostics.""" + + http_request: HttpRequest | None = None + http_response: HttpResponse | None = None + + +TASK_TYPES: dict[str, type[Task]] = { + WEBHOOK_SEND_WORKFLOW: WebhookDeliveryTask, +} class TaskFilter(BaseModel): diff --git a/infrahub_sdk/testing/schemas/animal.py b/infrahub_sdk/testing/schemas/animal.py index bbc9992b2..03925fdc3 100644 --- a/infrahub_sdk/testing/schemas/animal.py +++ b/infrahub_sdk/testing/schemas/animal.py @@ -7,6 +7,7 @@ AttributeKind, GenericSchema, NodeSchema, + RelationshipCardinality, RelationshipDirection, RelationshipKind, SchemaRoot, @@ -44,7 +45,7 @@ def schema_animal(self) -> GenericSchema: kind=RelationshipKind.GENERIC, optional=False, peer=TESTING_PERSON, - cardinality="one", + cardinality=RelationshipCardinality.ONE, identifier="person__animal", direction=RelationshipDirection.OUTBOUND, ), @@ -53,7 +54,7 @@ def schema_animal(self) -> GenericSchema: kind=RelationshipKind.GENERIC, optional=True, peer=TESTING_PERSON, - cardinality="one", + cardinality=RelationshipCardinality.ONE, identifier="person__animal_friend", direction=RelationshipDirection.OUTBOUND, ), @@ -109,7 +110,7 @@ def schema_person(self) -> NodeSchema: name="animals", peer=TESTING_ANIMAL, identifier="person__animal", - cardinality="many", + cardinality=RelationshipCardinality.MANY, direction=RelationshipDirection.INBOUND, max_count=10, ), @@ -117,21 +118,21 @@ def schema_person(self) -> NodeSchema: name="favorite_animal", peer=TESTING_ANIMAL, identifier="favorite_animal", - cardinality="one", + cardinality=RelationshipCardinality.ONE, direction=RelationshipDirection.INBOUND, ), Rel( name="best_friends", peer=TESTING_ANIMAL, identifier="person__animal_friend", - cardinality="many", + cardinality=RelationshipCardinality.MANY, direction=RelationshipDirection.INBOUND, ), Rel( name="tags", optional=True, peer=BUILTIN_TAG, - cardinality="many", + cardinality=RelationshipCardinality.MANY, ), ], ) diff --git a/infrahub_sdk/testing/schemas/car_person.py b/infrahub_sdk/testing/schemas/car_person.py index 3a1c3dc3c..99efe4088 100644 --- a/infrahub_sdk/testing/schemas/car_person.py +++ b/infrahub_sdk/testing/schemas/car_person.py @@ -5,7 +5,7 @@ import pytest -from infrahub_sdk.schema.main import AttributeKind, NodeSchema, RelationshipKind, SchemaRoot +from infrahub_sdk.schema.main import AttributeKind, NodeSchema, RelationshipCardinality, RelationshipKind, SchemaRoot from infrahub_sdk.schema.main import AttributeSchema as Attr from infrahub_sdk.schema.main import RelationshipSchema as Rel @@ -57,7 +57,13 @@ def schema_person_base(self) -> NodeSchema: Attr(name="age", kind=AttributeKind.NUMBER, optional=True), ], relationships=[ - Rel(name="cars", kind=RelationshipKind.GENERIC, optional=True, peer=TESTING_CAR, cardinality="many") + Rel( + name="cars", + kind=RelationshipKind.GENERIC, + optional=True, + peer=TESTING_CAR, + cardinality=RelationshipCardinality.MANY, + ) ], ) @@ -81,21 +87,21 @@ def schema_car_base(self) -> NodeSchema: kind=RelationshipKind.ATTRIBUTE, optional=False, peer=TESTING_PERSON, - cardinality="one", + cardinality=RelationshipCardinality.ONE, ), Rel( name="manufacturer", kind=RelationshipKind.ATTRIBUTE, optional=False, peer=TESTING_MANUFACTURER, - cardinality="one", + cardinality=RelationshipCardinality.ONE, identifier="car__manufacturer", ), Rel( name="tags", optional=True, peer=BUILTIN_TAG, - cardinality="many", + cardinality=RelationshipCardinality.MANY, ), ], ) @@ -118,7 +124,7 @@ def schema_manufacturer_base(self) -> NodeSchema: kind=RelationshipKind.GENERIC, optional=True, peer=TESTING_CAR, - cardinality="many", + cardinality=RelationshipCardinality.MANY, identifier="car__manufacturer", ), Rel( @@ -126,7 +132,7 @@ def schema_manufacturer_base(self) -> NodeSchema: kind=RelationshipKind.GENERIC, optional=True, peer=TESTING_PERSON, - cardinality="many", + cardinality=RelationshipCardinality.MANY, identifier="person__manufacturer", ), ], diff --git a/infrahub_sdk/transfer/importer/json.py b/infrahub_sdk/transfer/importer/json.py index 32928df97..8b0c1e730 100644 --- a/infrahub_sdk/transfer/importer/json.py +++ b/infrahub_sdk/transfer/importer/json.py @@ -6,7 +6,6 @@ from pathlib import Path from typing import TYPE_CHECKING, Any -import pyarrow.json as pa_json import ujson from rich.progress import Progress @@ -53,6 +52,17 @@ def wrapped_task_output(self, start: str, end: str = "[green]done") -> Generator self.console.print(f"{end}") async def import_data(self, import_directory: Path, branch: str) -> None: + # pyarrow is a heavy, optional dependency used only to read the line-delimited JSON + # export. Import it lazily so the rest of infrahubctl works without it being installed; + # only `infrahubctl object load` reaches this code path. + try: + import pyarrow.json as pa_json # noqa: PLC0415 + except ModuleNotFoundError as exc: + raise ModuleNotFoundError( + "Loading objects requires pyarrow, install the 'ctl' extra of the infrahub-sdk " + "package, `pip install 'infrahub-sdk[ctl]'` or run `uv sync --extra ctl`." + ) from exc + node_file = import_directory / "nodes.json" relationship_file = import_directory / "relationships.json" for f in (node_file, relationship_file): diff --git a/pyproject.toml b/pyproject.toml index 6a86188dc..b8941b9a7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "infrahub-sdk" -version = "1.22.3" +dynamic = ["version"] description = "Python Client to interact with Infrahub" authors = [ {name = "OpsMill", email = "info@opsmill.com"} @@ -47,6 +47,7 @@ ctl = [ "numpy>=1.26.2; python_version>='3.12'", "pyarrow>=14", "pyyaml>=6", + "ruamel.yaml>=0.18", "rich>=12,<14", "typer>=0.15.0", "click>=8.3,<9", @@ -61,6 +62,7 @@ all = [ "pyarrow>=14", "pytest", "pyyaml>=6", + "ruamel.yaml>=0.18", "rich>=12,<14", "typer>=0.15.0", "click>=8.3,<9", @@ -71,6 +73,7 @@ all = [ # Core optional dependencies tests = [ "infrahub-testcontainers>=1.7.3", + "jsonschema>=4.25.1", "pytest>=9.0,<9.1", "pytest-asyncio>=1.3,<1.4", "pytest-clarity>=1.0.1", @@ -275,6 +278,20 @@ disable_error_code = ["call-overload"] module = "infrahub_sdk.utils" disable_error_code = ["arg-type", "attr-defined", "return-value", "union-attr"] +[[tool.mypy.overrides]] +# ``main.py`` intentionally narrows the ``attributes``/``relationships``/``choices`` fields inherited +# from the generated write/read models to the user-facing constructible variants (``AttributeSchema``, +# ``RelationshipSchema``, and the ``*API`` read models). pydantic invariance makes mypy flag these +# deliberate overrides; the narrowed types are a superset-safe refinement validated at runtime. +module = "infrahub_sdk.schema.main" +disable_error_code = ["assignment"] + +[[tool.mypy.overrides]] +# The generated read models expose ``kind`` via ``@computed_field`` stacked on ``@property``. mypy +# does not support decorators on top of ``@property`` and flags it, but pydantic requires this order. +module = "infrahub_sdk.schema.generated.read" +disable_error_code = ["misc"] + [tool.ruff] line-length = 120 @@ -304,8 +321,6 @@ ignore = [ "CPY", # flake8-copyright "T201", # use of `print` "COM812", # missing-trailing-comma - "D203", # incorrect-blank-line-before-class (incompatible with D211) - "D213", # multi-line-summary-second-line (incompatible with D212) ################################################################################################## # Rules below needs to be Investigated # @@ -350,13 +365,9 @@ ignore = [ "D104", # Missing docstring in public package "D105", # Missing docstring in magic method "D107", # Missing docstring in `__init__` - "D301", # Use `r"""` if any backslashes in a docstring - "D401", # First line of docstring should be in imperative mood - "D404", # First word of the docstring should not be "This" + "D301", # Use `r"""` if any backslashes in a docstring — conflicts with Typer's `\b` no-wrap marker in CLI command docstrings, which must stay a real escape (not a raw string) "D417", # Missing argument description in the docstring - "DOC102", # Docstring contains extraneous parameter(s) "DOC201", # `return` is not documented in docstring - "DOC402", # `yield` is not documented in docstring "DOC502", # Raised exception is not explicitly raised (false positives for transitive raises through helpers) ] @@ -382,6 +393,10 @@ ignorelist = [ [tool.ruff.lint.isort] known-first-party = ["infrahub_sdk", "infrahub_ctl"] +[tool.ruff.lint.pydocstyle] +# Also lets the DOC (pydoclint) rules parse Args/Returns/Yields sections correctly. +convention = "google" + [tool.ruff.lint.pycodestyle] max-line-length = 150 @@ -391,6 +406,15 @@ max-complexity = 14 [tool.ruff.lint.per-file-ignores] +"infrahub_sdk/schema/generated/*.py" = [ + # Generated pydantic models use `from __future__ import annotations`, so the enum imports + # they reference are required at runtime for pydantic to resolve the annotations; moving + # them into a type-checking block would break model construction. + "TC001", # Move application import into a type-checking block + # The AttributeKind enum carries a member whose value is the literal string "Password". + "S105", # Possible hardcoded password assigned to a variable +] + "infrahub_sdk/**/*.py" = [ ################################################################################################## # Review and change the below later # @@ -549,8 +573,25 @@ front_matter_title = "" prohibited_texts = [] [build-system] -requires = ["hatchling"] +requires = ["hatchling", "hatch-vcs"] build-backend = "hatchling.build" +[tool.hatch.version] +source = "vcs" +# Static sentinel for tag-less checkouts (e.g. a shallow clone without tags). It is identical on +# every branch so releases never touch it — a moving value would conflict on every stable/develop +# merge, which is exactly what dynamic versioning removes. Tag-less builds resolve 0.0.0.devN+g, +# unmistakably a non-release, and are publish-guarded in release.yml. +fallback-version = "0.0.0.dev0" + +[tool.hatch.version.raw-options] +# No --dirty: keeps the resolved version derived from committed git state only, independent of the +# work tree contents, so a build from a dirty/stripped tree never bumps an on-tag build to a dev +# version. +git_describe_command = ["git", "describe", "--tags", "--long", "--match", "v*"] + +[tool.hatch.build.hooks.vcs] +version-file = "infrahub_sdk/_version.py" + [tool.hatch.build.targets.wheel] packages = ["infrahub_sdk"] diff --git a/tasks.py b/tasks.py index 69bf79860..b29b1d695 100644 --- a/tasks.py +++ b/tasks.py @@ -427,3 +427,41 @@ def generate_repository_jsonschema(context: Context) -> None: repository_jsonschema.parent.mkdir(parents=True, exist_ok=True) repository_jsonschema.write_text(schema) print(f"Wrote to {repository_jsonschema}") + + +@task(name="schema-drift-check") +def schema_drift_check(context: Context) -> None: # noqa: ARG001 + """Warn (without failing) if the live Infrahub JSON schema drifted from the committed baseline. + + Emits GitHub Actions ``::warning::`` annotations for any added or removed + schema property so the formatter's canonical key ordering in + ``infrahub_sdk/ctl/schema_format.py`` can be updated. Always exits 0. + """ + from infrahub_sdk.ctl.schema_drift import compute_drift, fetch_live_properties, load_baseline + + try: + live = fetch_live_properties() + except Exception as exc: + print(f"::warning title=Schema drift check::Could not fetch the Infrahub schema: {exc}") + return + + drift = compute_drift(live=live, baseline=load_baseline()) + if not drift: + print("Infrahub schema is in sync with the committed baseline; no drift detected.") + return + + hint = "update infrahub_sdk/ctl/schema_format.py if needed, then run 'invoke schema-drift-update'" + for definition, change in drift.items(): + for prop in change["added"]: + print(f"::warning title=Schema drift::New schema property {definition}.{prop} — {hint}") + for prop in change["removed"]: + print(f"::warning title=Schema drift::Removed schema property {definition}.{prop} — {hint}") + + +@task(name="schema-drift-update") +def schema_drift_update(context: Context) -> None: # noqa: ARG001 + """Refresh infrahub_sdk/ctl/schema_properties.json from the live Infrahub JSON schema.""" + from infrahub_sdk.ctl.schema_drift import BASELINE_PATH, fetch_live_properties, write_baseline + + write_baseline(fetch_live_properties()) + print(f"Updated {BASELINE_PATH}") diff --git a/tests/fixtures/models/valid_schemas/contract.yml b/tests/fixtures/models/valid_schemas/contract.yml index 398f2bd7f..0b9ca690b 100644 --- a/tests/fixtures/models/valid_schemas/contract.yml +++ b/tests/fixtures/models/valid_schemas/contract.yml @@ -18,7 +18,7 @@ nodes: kind: Text optional: true relationships: - - name: Organization + - name: organization peer: TestOrganization optional: false cardinality: one diff --git a/tests/fixtures/schema_01.json b/tests/fixtures/schema_01.json index c2fab38ad..8504edcae 100644 --- a/tests/fixtures/schema_01.json +++ b/tests/fixtures/schema_01.json @@ -8,7 +8,7 @@ "attributes": [ { "name": "query", - "kind": "String", + "kind": "Text", "label": null, "description": null, "default_value": null, @@ -19,7 +19,7 @@ }, { "name": "description", - "kind": "String", + "kind": "Text", "label": null, "description": null, "default_value": null, @@ -30,7 +30,7 @@ }, { "name": "name", - "kind": "String", + "kind": "Text", "label": null, "description": null, "default_value": null, @@ -65,7 +65,7 @@ "attributes": [ { "name": "username", - "kind": "String", + "kind": "Text", "label": null, "description": null, "default_value": null, @@ -76,7 +76,7 @@ }, { "name": "type", - "kind": "String", + "kind": "Text", "label": null, "description": null, "default_value": "LOCAL", @@ -87,7 +87,7 @@ }, { "name": "name", - "kind": "String", + "kind": "Text", "label": null, "description": null, "default_value": null, @@ -98,7 +98,7 @@ }, { "name": "description", - "kind": "String", + "kind": "Text", "label": null, "description": null, "default_value": null, @@ -109,7 +109,7 @@ }, { "name": "commit", - "kind": "String", + "kind": "Text", "label": null, "description": null, "default_value": null, @@ -120,7 +120,7 @@ }, { "name": "location", - "kind": "String", + "kind": "Text", "label": null, "description": null, "default_value": null, @@ -131,7 +131,7 @@ }, { "name": "password", - "kind": "String", + "kind": "Text", "label": null, "description": null, "default_value": null, @@ -142,7 +142,7 @@ }, { "name": "default_branch", - "kind": "String", + "kind": "Text", "label": null, "description": null, "default_value": "main", @@ -194,7 +194,7 @@ "attributes": [ { "name": "name", - "kind": "String", + "kind": "Text", "label": null, "description": null, "default_value": null, @@ -205,7 +205,7 @@ }, { "name": "description", - "kind": "String", + "kind": "Text", "label": null, "description": null, "default_value": null, @@ -254,17 +254,17 @@ "attributes": [ { "name": "name", - "kind": "String", + "kind": "Text", "unique": true }, { "name": "description", - "kind": "String", + "kind": "Text", "optional": true }, { "name": "type", - "kind": "String" + "kind": "Text" } ], "relationships": [ diff --git a/tests/fixtures/schema_02.json b/tests/fixtures/schema_02.json index 1417053f1..bcdde5373 100644 --- a/tests/fixtures/schema_02.json +++ b/tests/fixtures/schema_02.json @@ -683,7 +683,7 @@ "attributes": [ { "name": "query", - "kind": "String", + "kind": "Text", "label": null, "description": null, "default_value": null, @@ -694,7 +694,7 @@ }, { "name": "description", - "kind": "String", + "kind": "Text", "label": null, "description": null, "default_value": null, @@ -705,7 +705,7 @@ }, { "name": "name", - "kind": "String", + "kind": "Text", "label": null, "description": null, "default_value": null, @@ -740,7 +740,7 @@ "attributes": [ { "name": "username", - "kind": "String", + "kind": "Text", "label": null, "description": null, "default_value": null, @@ -751,7 +751,7 @@ }, { "name": "type", - "kind": "String", + "kind": "Text", "label": null, "description": null, "default_value": "LOCAL", @@ -762,7 +762,7 @@ }, { "name": "name", - "kind": "String", + "kind": "Text", "label": null, "description": null, "default_value": null, @@ -773,7 +773,7 @@ }, { "name": "description", - "kind": "String", + "kind": "Text", "label": null, "description": null, "default_value": null, @@ -784,7 +784,7 @@ }, { "name": "commit", - "kind": "String", + "kind": "Text", "label": null, "description": null, "default_value": null, @@ -795,7 +795,7 @@ }, { "name": "location", - "kind": "String", + "kind": "Text", "label": null, "description": null, "default_value": null, @@ -806,7 +806,7 @@ }, { "name": "password", - "kind": "String", + "kind": "Text", "label": null, "description": null, "default_value": null, @@ -817,7 +817,7 @@ }, { "name": "default_branch", - "kind": "String", + "kind": "Text", "label": null, "description": null, "default_value": "main", @@ -884,7 +884,7 @@ "attributes": [ { "name": "name", - "kind": "String", + "kind": "Text", "label": null, "description": null, "default_value": null, @@ -895,7 +895,7 @@ }, { "name": "description", - "kind": "String", + "kind": "Text", "label": null, "description": null, "default_value": null, @@ -923,17 +923,17 @@ "attributes": [ { "name": "name", - "kind": "String", + "kind": "Text", "unique": true }, { "name": "description", - "kind": "String", + "kind": "Text", "optional": true }, { "name": "type", - "kind": "String" + "kind": "Text" } ], "relationships": [ diff --git a/tests/unit/ctl/conftest.py b/tests/unit/ctl/conftest.py index b4089388a..b11acc6ce 100644 --- a/tests/unit/ctl/conftest.py +++ b/tests/unit/ctl/conftest.py @@ -5,7 +5,7 @@ from pytest_httpx import HTTPXMock from infrahub_sdk.utils import get_fixtures_dir -from tests.unit.sdk.conftest import mock_query_infrahub_user, mock_query_infrahub_version # noqa: F401 +from tests.unit.sdk.conftest import mock_query_infrahub_server_info, mock_query_infrahub_user # noqa: F401 @pytest.fixture diff --git a/tests/unit/ctl/test_cli.py b/tests/unit/ctl/test_cli.py index 410646450..12a446e67 100644 --- a/tests/unit/ctl/test_cli.py +++ b/tests/unit/ctl/test_cli.py @@ -33,10 +33,10 @@ def test_version_command() -> None: assert "Python SDK: v" in result.stdout -def test_info_command_success(mock_query_infrahub_version: HTTPXMock, mock_query_infrahub_user: HTTPXMock) -> None: +def test_info_command_success(mock_query_infrahub_server_info: HTTPXMock, mock_query_infrahub_user: HTTPXMock) -> None: result = runner.invoke(app, ["info"], env={"INFRAHUB_API_TOKEN": "foo"}) assert result.exit_code == 0 - for expected in ["Connection Status", "Python Version", "SDK Version", "Infrahub Version"]: + for expected in ["Connection Status", "Python Version", "SDK Version", "Infrahub Version", "Deployment ID"]: assert expected in result.stdout, f"'{expected}' not found in info command output" @@ -47,7 +47,7 @@ def test_info_command_failure() -> None: def test_info_detail_command_success( - mock_query_infrahub_version: HTTPXMock, mock_query_infrahub_user: HTTPXMock + mock_query_infrahub_server_info: HTTPXMock, mock_query_infrahub_user: HTTPXMock ) -> None: result = runner.invoke(app, ["info", "--detail"], env={"INFRAHUB_API_TOKEN": "foo"}) assert result.exit_code == 0 @@ -55,7 +55,7 @@ def test_info_detail_command_success( assert expected in result.stdout, f"'{expected}' not found in detailed info command output" -def test_anonymous_info_detail_command_success(mock_query_infrahub_version: HTTPXMock) -> None: +def test_anonymous_info_detail_command_success(mock_query_infrahub_server_info: HTTPXMock) -> None: result = runner.invoke(app, ["info", "--detail"]) assert result.exit_code == 0 for expected in ["Connection Status", "Version Information", "Client Info", "Infrahub Info", "anonymous"]: diff --git a/tests/unit/ctl/test_schema_app.py b/tests/unit/ctl/test_schema_app.py index 8be46056a..bbd498b54 100644 --- a/tests/unit/ctl/test_schema_app.py +++ b/tests/unit/ctl/test_schema_app.py @@ -81,53 +81,24 @@ def test_schema_load_multiple(httpx_mock: HTTPXMock) -> None: assert content_json == {"schemas": [fixture_file1_content, fixture_file2_content]} -def test_schema_load_notvalid_namespace(httpx_mock: HTTPXMock) -> None: +def test_schema_load_notvalid_namespace() -> None: + """An invalid namespace is now rejected client-side by the write contract. + + The SDK write models mirror the server's field constraints, so ``infrahubctl load`` + catches an invalid namespace during local validation and exits before sending the + payload to the server. + """ fixture_file = get_fixtures_dir() / "models" / "non_valid_namespace.json" - httpx_mock.add_response( - method="POST", - url="http://mock/api/schema/load?branch=main", - status_code=422, - json={ - "detail": [ - { - "type": "string_pattern_mismatch", - "loc": ["body", "schemas", 0, "nodes", 0, "namespace"], - "msg": "String should match pattern '^[A-Z][a-z0-9]+$'", - "input": "OuT", - "ctx": {"pattern": "^[A-Z][a-z0-9]+$"}, - "url": "https://errors.pydantic.dev/2.7/v/string_pattern_mismatch", - }, - { - "type": "value_error", - "loc": ["body", "schemas", 0, "nodes", 0, "attributes", 0, "kind"], - "msg": "Value error, Only valid Attribute Kind are : ['ID', 'Dropdown'] ", - "input": "NotValid", - "ctx": {"error": {}}, - "url": "https://errors.pydantic.dev/2.7/v/value_error", - }, - ] - }, - ) result = runner.invoke(app=app, args=["load", str(fixture_file)]) assert result.exit_code == 1 clean_output = remove_ansi_color(result.stdout.replace("\n", "")) - expected_result = ( - "Unable to load the schema: Node: OuTDevice | " - "namespace (OuT) | String should match pattern '^[A-Z][a-z0-9]+$' (string_pattern_mismatch) " - " Node: OuTDevice | Attribute: name (NotValid) | Value error, Only valid Attribute Kind " - "are : ['ID', 'Dropdown'] (value_error)" - ) - assert expected_result == clean_output - - content = httpx_mock.get_requests()[0].content.decode("utf8") - content_json = yaml.safe_load(content) - fixture_file_content = yaml.safe_load( - fixture_file.read_text(encoding="utf-8"), - ) - assert content_json == {"schemas": [fixture_file_content]} + assert "Schema not valid" in clean_output + assert "nodes[0].namespace" in clean_output + assert "String should match pattern" in clean_output + assert "received: 'OuT'" in clean_output def test_load_valid_generic_schema(httpx_mock: HTTPXMock) -> None: diff --git a/tests/unit/ctl/test_schema_drift.py b/tests/unit/ctl/test_schema_drift.py new file mode 100644 index 000000000..bdad91145 --- /dev/null +++ b/tests/unit/ctl/test_schema_drift.py @@ -0,0 +1,79 @@ +"""Unit tests for the schema drift-detection logic (offline).""" + +from __future__ import annotations + +import json +from typing import TYPE_CHECKING + +from infrahub_sdk.ctl.schema_drift import ( + BASELINE_PATH, + SCHEMA_URL, + TRACKED_DEFINITIONS, + compute_drift, + extract_properties, + fetch_live_properties, + load_baseline, + write_baseline, +) + +if TYPE_CHECKING: + from pathlib import Path + + from pytest_httpx import HTTPXMock + + +def test_extract_properties_reads_defs() -> None: + schema = { + "$defs": { + "NodeSchema": {"properties": {"name": {}, "namespace": {}}}, + "AttributeSchema": {"properties": {"kind": {}, "name": {}}}, + } + } + result = extract_properties(schema) + + # Every tracked definition is present; values are sorted; unknown defs empty. + assert set(result) == set(TRACKED_DEFINITIONS) + assert result["NodeSchema"] == ["name", "namespace"] + assert result["AttributeSchema"] == ["kind", "name"] + assert result["RelationshipSchema"] == [] + + +def test_compute_drift_detects_added_and_removed() -> None: + baseline = {"NodeSchema": ["name", "namespace", "label"]} + live = {"NodeSchema": ["name", "namespace", "new_field"]} + + drift = compute_drift(live=live, baseline=baseline) + + assert drift == {"NodeSchema": {"added": ["new_field"], "removed": ["label"]}} + + +def test_compute_drift_empty_when_in_sync() -> None: + props = {"NodeSchema": ["name", "namespace"]} + assert compute_drift(live=props, baseline=props) == {} + + +def test_committed_baseline_is_valid_and_complete() -> None: + baseline = load_baseline() + # The shipped baseline covers exactly the tracked definitions and is JSON. + assert set(baseline) == set(TRACKED_DEFINITIONS) + assert all(isinstance(props, list) for props in baseline.values()) + # Round-trips through json (guards against a hand-edit breaking the file). + assert json.loads(BASELINE_PATH.read_text(encoding="utf-8")) == baseline + + +def test_write_and_load_baseline_roundtrip(tmp_path: Path) -> None: + path = tmp_path / "baseline.json" + data = {"NodeSchema": ["name", "namespace"], "AttributeSchema": ["kind", "name"]} + write_baseline(data, path) + assert load_baseline(path) == data + + +def test_fetch_live_properties_extracts_from_response(httpx_mock: HTTPXMock) -> None: + httpx_mock.add_response( + url=SCHEMA_URL, + json={"$defs": {"NodeSchema": {"properties": {"namespace": {}, "name": {}}}}}, + ) + props = fetch_live_properties() + # Sorted names for the tracked definition; other tracked defs default to []. + assert props["NodeSchema"] == ["name", "namespace"] + assert props["RelationshipSchema"] == [] diff --git a/tests/unit/ctl/test_schema_format.py b/tests/unit/ctl/test_schema_format.py new file mode 100644 index 000000000..b862f3fdd --- /dev/null +++ b/tests/unit/ctl/test_schema_format.py @@ -0,0 +1,474 @@ +"""Unit tests for the pure schema-formatting logic in ``schema_format``.""" + +from __future__ import annotations + +from collections import OrderedDict + +import pytest +import yaml + +from infrahub_sdk.ctl.schema_format import ( + FormatError, + FormatOptions, + format_schema_text, + is_schema_document, + reorder_mapping, +) + +NODE_DOC = """\ +--- +# yaml-language-server: $schema=https://schema.infrahub.app/infrahub/schema/latest.json +version: "1.0" + +nodes: + - relationships: + - peer: BuiltinTag + name: tags + attributes: + - order_weight: 1500 + optional: true + name: status + kind: Dropdown + choices: + - color: "#fff" + name: active + label: Active + namespace: Dcim + name: Device + label: Device + description: A device. +""" + + +def _keys_of(text: str, path: list) -> list[str]: + """Load formatted YAML and return the key order of the mapping at ``path``.""" + data = yaml.safe_load(text) + for step in path: + data = data[step] + return list(data.keys()) + + +def test_reorder_mapping_leading_trailing_and_unknown() -> None: + data = OrderedDict([("order_weight", 1000), ("extra", "x"), ("kind", "Text"), ("name", "field")]) + reorder_mapping(data, leading=["name", "kind"], trailing=["order_weight"]) + + # name/kind first, order_weight last, unknown key preserved in the middle. + assert list(data.keys()) == ["name", "kind", "extra", "order_weight"] + + +def test_node_key_order_is_canonical() -> None: + text = format_schema_text(NODE_DOC) + + # Top-level sections: version before nodes. + assert _keys_of(text, [])[:2] == ["version", "nodes"] + # name/namespace first; attributes then relationships always last. + assert _keys_of(text, ["nodes", 0]) == [ + "name", + "namespace", + "description", + "label", + "attributes", + "relationships", + ] + + +def test_attribute_relationship_and_choice_inner_order() -> None: + text = format_schema_text(NODE_DOC) + + attr_keys = _keys_of(text, ["nodes", 0, "attributes", 0]) + assert attr_keys == ["name", "kind", "choices", "optional", "order_weight"] + assert attr_keys[-1] == "order_weight" + + assert _keys_of(text, ["nodes", 0, "attributes", 0, "choices", 0]) == ["name", "label", "color"] + assert _keys_of(text, ["nodes", 0, "relationships", 0]) == ["name", "peer"] + + +def test_restricted_namespace_nodes_are_untouched() -> None: + doc = """\ +--- +version: "1.0" +nodes: + - namespace: Core + name: Something + attributes: + - order_weight: 1 + kind: Text + name: x + - namespace: Dcim + name: Device + attributes: + - order_weight: 1 + kind: Text + name: x +""" + text = format_schema_text(doc) + + # Core node keeps its authored (scrambled) attribute key order. + assert _keys_of(text, ["nodes", 0, "attributes", 0]) == ["order_weight", "kind", "name"] + # Dcim (user) node is reordered. + assert _keys_of(text, ["nodes", 1, "attributes", 0]) == ["name", "kind", "order_weight"] + + +def test_extensions_are_formatted() -> None: + doc = """\ +--- +version: "1.0" +extensions: + nodes: + - relationships: + - peer: LocationSite + name: sites + kind: OrganizationProvider +""" + text = format_schema_text(doc) + assert _keys_of(text, ["extensions", "nodes", 0]) == ["kind", "relationships"] + assert _keys_of(text, ["extensions", "nodes", 0, "relationships", 0]) == ["name", "peer"] + + +def test_unknown_keys_are_preserved_not_dropped() -> None: + doc = """\ +--- +version: "1.0" +nodes: + - name: Device + namespace: Dcim + some_future_key: value +""" + text = format_schema_text(doc) + assert _keys_of(text, ["nodes", 0]) == ["name", "namespace", "some_future_key"] + assert yaml.safe_load(text)["nodes"][0]["some_future_key"] == "value" + + +def test_comments_are_preserved() -> None: + doc = """\ +--- +# yaml-language-server: $schema=https://schema.infrahub.app/infrahub/schema/latest.json +version: "1.0" + +nodes: + # a banner comment before the node + - namespace: Dcim + name: Device + attributes: + - name: status + kind: Dropdown + choices: + - name: active + color: "#7fbf7f" # a trailing inline comment +""" + text = format_schema_text(doc) + assert "# a banner comment before the node" in text + assert "# a trailing inline comment" in text + assert "yaml-language-server" in text + + +def test_flow_style_sequences_are_preserved() -> None: + doc = """\ +--- +version: "1.0" +nodes: + - name: Device + namespace: Dcim + uniqueness_constraints: + - [manufacturer, name__value] +""" + text = format_schema_text(doc) + # The inline (flow) sequence is not expanded to block style. + assert "[manufacturer, name__value]" in text + + +def test_quotes_are_preserved() -> None: + doc = """\ +--- +version: "1.0" +nodes: + - name: Device + namespace: Dcim + description: "A quoted description" +""" + text = format_schema_text(doc) + assert 'description: "A quoted description"' in text + assert 'version: "1.0"' in text + + +def test_multiline_string_round_trips() -> None: + doc = """\ +--- +version: "1.0" +nodes: + - name: Device + namespace: Dcim + attributes: + - name: computed + kind: Text + read_only: true + computed_attribute: + kind: Jinja2 + jinja2_template: >- + {{ a__value }}-{{ b__value }} +""" + text = format_schema_text(doc) + assert yaml.safe_load(text) == yaml.safe_load(doc) + + +def test_header_is_added_when_missing() -> None: + doc = """\ +--- +version: "1.0" +nodes: + - name: Device + namespace: Dcim +""" + text = format_schema_text(doc) + assert text.startswith("---\n# yaml-language-server:") + + +def test_format_is_idempotent_and_semantics_preserved() -> None: + once = format_schema_text(NODE_DOC) + assert format_schema_text(once) == once + assert yaml.safe_load(once) == yaml.safe_load(NODE_DOC) + + +def test_non_schema_document_is_returned_unchanged() -> None: + doc = "apiVersion: infrahub.app/v1\nkind: Menu\nspec:\n data: []\n" + assert format_schema_text(doc) == doc + + +def test_format_error_raised_on_semantic_drift(monkeypatch: pytest.MonkeyPatch) -> None: + # Simulate a formatting step that silently drops data; the guard must catch it. + def _wipe(data: dict, options: object = None) -> None: + data.clear() + + monkeypatch.setattr("infrahub_sdk.ctl.schema_format.format_document", _wipe) + + with pytest.raises(FormatError): + format_schema_text(NODE_DOC) + + +def test_malformed_non_list_attributes_is_left_untouched() -> None: + # A parseable schema whose `attributes` is not a list must not crash the + # formatter; that section is simply left as-is. + doc = """\ +--- +version: "1.0" +nodes: + - namespace: Dcim + name: Device + attributes: 5 + relationships: not-a-list +""" + text = format_schema_text(doc) + assert yaml.safe_load(text) == yaml.safe_load(doc) + + +def test_header_not_added_when_substring_appears_in_scalar() -> None: + # `yaml-language-server` appearing in a value (not as a real header line) + # must not suppress the header being added. + doc = """\ +--- +version: "1.0" +nodes: + - namespace: Dcim + name: Device + description: see the yaml-language-server extension docs +""" + text = format_schema_text(doc) + assert text.startswith("---\n# yaml-language-server: $schema=") + # The real directive appears exactly once (added, not duplicated later). + assert text.count("# yaml-language-server:") == 1 + + +def test_existing_header_is_not_duplicated() -> None: + doc = """\ +--- +# yaml-language-server: $schema=https://schema.infrahub.app/infrahub/schema/latest.json +version: "1.0" +nodes: + - namespace: Dcim + name: Device +""" + text = format_schema_text(doc) + assert text.count("# yaml-language-server:") == 1 + + +STRIP_DOC = """\ +--- +version: "1.0" +nodes: + - namespace: Dcim + name: Device + attributes: + - name: a + kind: Text + optional: false + - name: b + kind: Text + optional: true + relationships: + - name: r1 + peer: DcimX + optional: true + cardinality: many + kind: Generic + - name: r2 + peer: DcimY + optional: false + cardinality: one + kind: Attribute +""" + + +def test_strip_defaults_removes_only_default_values() -> None: + node = yaml.safe_load(format_schema_text(STRIP_DOC, FormatOptions(strip_defaults=True)))["nodes"][0] + attrs = {a["name"]: a for a in node["attributes"]} + rels = {r["name"]: r for r in node["relationships"]} + + # Attribute default optional:false stripped; non-default optional:true kept. + assert "optional" not in attrs["a"] + assert attrs["b"]["optional"] is True + + # Relationship defaults (optional:true, cardinality:many, kind:Generic) stripped. + assert set(rels["r1"].keys()) == {"name", "peer"} + # Non-default relationship values are kept. + assert rels["r2"]["optional"] is False + assert rels["r2"]["cardinality"] == "one" + assert rels["r2"]["kind"] == "Attribute" + + +SORT_DOC = """\ +--- +version: "1.0" +nodes: + - namespace: Dcim + name: Device + attributes: + - name: c + kind: Text + order_weight: 3000 + - name: a + kind: Text + order_weight: 1000 + - name: b + kind: Text + - name: d + kind: Text + order_weight: 2000 +""" + + +def test_sort_by_order_weight_ascending_missing_last() -> None: + node = yaml.safe_load(format_schema_text(SORT_DOC, FormatOptions(sort_by_order_weight=True)))["nodes"][0] + names = [a["name"] for a in node["attributes"]] + # Weighted ascending (a=1000, d=2000, c=3000), then the weightless one last. + assert names == ["a", "d", "c", "b"] + + +def test_sort_permits_same_named_items_with_different_weights() -> None: + # The guard must neutralise the reorder by full content, not by name — two + # items sharing a name but differing in weight should sort, not abort. + doc = """\ +--- +version: "1.0" +nodes: + - namespace: Dcim + name: Device + attributes: + - name: dup + kind: Text + order_weight: 2000 + - name: dup + kind: Number + order_weight: 1000 +""" + text = format_schema_text(doc, FormatOptions(sort_by_order_weight=True)) + attrs = yaml.safe_load(text)["nodes"][0]["attributes"] + assert [a["order_weight"] for a in attrs] == [1000, 2000] + + +def test_backfill_order_weight_only_fills_missing() -> None: + doc = """\ +--- +version: "1.0" +nodes: + - namespace: Dcim + name: Device + attributes: + - name: a + kind: Text + - name: b + kind: Text + order_weight: 5 +""" + node = yaml.safe_load(format_schema_text(doc, FormatOptions(backfill_order_weight=True)))["nodes"][0] + weights = {a["name"]: a["order_weight"] for a in node["attributes"]} + assert weights == {"a": 1000, "b": 5} + + +def test_flags_are_idempotent_and_off_by_default() -> None: + # Off by default: no content change beyond ordering (STRIP_DOC has a + # strippable default that must survive when the flag is not set). + default_out = yaml.safe_load(format_schema_text(STRIP_DOC)) + assert default_out["nodes"][0]["attributes"][0].get("optional") is False + + opts = FormatOptions(strip_defaults=True, sort_by_order_weight=True, backfill_order_weight=True) + once = format_schema_text(STRIP_DOC, opts) + assert format_schema_text(once, opts) == once + + +def test_reorder_mapping_ignores_non_mapping() -> None: + # A scalar/None has no move_to_end; the call must be a harmless no-op. + reorder_mapping("not a mapping", ["name"], []) + reorder_mapping(None, ["name"], []) + + +def test_non_dict_list_items_and_nodes_are_left_untouched() -> None: + doc = """\ +--- +version: "1.0" +nodes: + - namespace: Dcim + name: Device + attributes: + - just_a_string + - a_scalar_node +""" + opts = FormatOptions(strip_defaults=True, sort_by_order_weight=True, backfill_order_weight=True) + text = format_schema_text(doc, opts) + assert yaml.safe_load(text) == yaml.safe_load(doc) + + +def test_extensions_with_non_dict_node_left_untouched() -> None: + doc = """\ +--- +version: "1.0" +extensions: + nodes: + - a_scalar_entry +""" + assert yaml.safe_load(format_schema_text(doc)) == yaml.safe_load(doc) + + +def test_duplicate_key_raises_format_error() -> None: + # Round-trip YAML rejects duplicate keys; it must surface as a FormatError + # (a per-file error), not ruamel's YAMLError leaking to the caller. + doc = """\ +--- +version: "1.0" +nodes: + - namespace: Dcim + name: Device + label: A + label: B +""" + with pytest.raises(FormatError): + format_schema_text(doc) + + +def test_is_schema_document() -> None: + assert is_schema_document({"version": "1.0", "nodes": []}) + assert is_schema_document({"version": "1.0", "generics": []}) + assert is_schema_document({"version": "1.0", "extensions": {}}) + assert not is_schema_document({"version": "1.0"}) + assert not is_schema_document({"nodes": []}) + assert not is_schema_document({"apiVersion": "infrahub.app/v1", "kind": "Menu"}) + assert not is_schema_document("not a dict") diff --git a/tests/unit/ctl/test_schema_format_app.py b/tests/unit/ctl/test_schema_format_app.py new file mode 100644 index 000000000..a7e1d439d --- /dev/null +++ b/tests/unit/ctl/test_schema_format_app.py @@ -0,0 +1,254 @@ +"""CLI tests for ``infrahubctl schema format``.""" + +from __future__ import annotations + +from pathlib import Path +from typing import TYPE_CHECKING + +import yaml +from typer.testing import CliRunner + +from infrahub_sdk.ctl import schema as schema_module +from infrahub_sdk.ctl.schema import app +from infrahub_sdk.ctl.schema_format import FormatError +from tests.helpers.cli import remove_ansi_color + +if TYPE_CHECKING: + import pytest + +runner = CliRunner() + +# Widen the Rich console so long tmp_path locations are not wrapped across +# lines, which would break substring assertions on the output. +WIDE = {"COLUMNS": "300"} + + +UNFORMATTED = """\ +--- +version: "1.0" +nodes: + - namespace: Dcim + name: Device + # a design note + label: Device + attributes: + - order_weight: 1000 + kind: Text + name: name + unique: true +""" + + +def _write(path: Path, content: str) -> Path: + path.write_text(content, encoding="utf-8") + return path + + +def test_format_writes_in_place(tmp_path: Path) -> None: + schema = _write(tmp_path / "dcim.yml", UNFORMATTED) + + result = runner.invoke(app, env=WIDE, args=["format", str(schema)]) + + assert result.exit_code == 0 + output = remove_ansi_color(result.stdout) + assert f"Reformatted {schema}" in output + assert "1 file(s) reformatted" in output + + formatted = schema.read_text(encoding="utf-8") + # Header re-added, keys reordered (name before kind, order_weight last). + assert formatted.startswith("---\n# yaml-language-server:") + name_idx = formatted.index("name: name") + kind_idx = formatted.index("kind: Text") + weight_idx = formatted.index("order_weight: 1000") + assert name_idx < kind_idx < weight_idx + + +def test_format_is_idempotent(tmp_path: Path) -> None: + schema = _write(tmp_path / "dcim.yml", UNFORMATTED) + + runner.invoke(app, env=WIDE, args=["format", str(schema)]) + once = schema.read_text(encoding="utf-8") + runner.invoke(app, env=WIDE, args=["format", str(schema)]) + twice = schema.read_text(encoding="utf-8") + + assert once == twice + + +def test_format_check_reports_and_exits_nonzero(tmp_path: Path) -> None: + schema = _write(tmp_path / "dcim.yml", UNFORMATTED) + before = schema.read_text(encoding="utf-8") + + result = runner.invoke(app, env=WIDE, args=["format", str(schema), "--check"]) + + assert result.exit_code == 1 + assert "Would reformat" in remove_ansi_color(result.stdout) + # --check never writes. + assert schema.read_text(encoding="utf-8") == before + + +def test_format_check_clean_file_exits_zero(tmp_path: Path) -> None: + schema = _write(tmp_path / "dcim.yml", UNFORMATTED) + runner.invoke(app, env=WIDE, args=["format", str(schema)]) # normalise first + + result = runner.invoke(app, env=WIDE, args=["format", str(schema), "--check"]) + + assert result.exit_code == 0 + assert "0 file(s) would be reformatted" in remove_ansi_color(result.stdout) + + +def test_format_diff_prints_and_does_not_write(tmp_path: Path) -> None: + schema = _write(tmp_path / "dcim.yml", UNFORMATTED) + before = schema.read_text(encoding="utf-8") + + result = runner.invoke(app, env=WIDE, args=["format", str(schema), "--diff"]) + + assert result.exit_code == 0 + output = remove_ansi_color(result.stdout) + assert "yaml-language-server" in output # the added header shows up in the diff + # Colour is applied via Rich styling, not literal markup tags. + assert "[green]" not in output + assert "[red]" not in output + assert schema.read_text(encoding="utf-8") == before + + +def test_format_preserves_comments(tmp_path: Path) -> None: + schema = _write(tmp_path / "dcim.yml", UNFORMATTED) + + result = runner.invoke(app, env=WIDE, args=["format", str(schema)]) + + assert result.exit_code == 0 + # The comment survives the reformat. + assert "# a design note" in schema.read_text(encoding="utf-8") + + +def test_format_skips_multi_document_file(tmp_path: Path) -> None: + multi = _write( + tmp_path / "multi.yml", + '---\nversion: "1.0"\nnodes: []\n---\nversion: "1.0"\ngenerics: []\n', + ) + + result = runner.invoke(app, env=WIDE, args=["format", str(multi)]) + + assert result.exit_code == 0 + output = remove_ansi_color(result.stdout) + assert "multi-document files are not supported" in output + # Left untouched. + assert multi.read_text(encoding="utf-8").count("---") == 2 + + +def test_format_reports_invalid_file(tmp_path: Path) -> None: + bad = _write(tmp_path / "bad.yml", 'version: "1.0"\nnodes: [unclosed\n') + + result = runner.invoke(app, env=WIDE, args=["format", str(bad)]) + + assert result.exit_code == 1 + + +def test_format_duplicate_key_is_per_file_error(tmp_path: Path) -> None: + # A duplicate key (which `schema load`/PyYAML tolerate) must be reported as + # a per-file error without aborting the run: other files still format. + _write( + tmp_path / "a_dup.yml", + '---\nversion: "1.0"\nnodes:\n - namespace: Dcim\n name: Device\n label: A\n label: B\n', + ) + good = _write(tmp_path / "b_good.yml", UNFORMATTED) + + result = runner.invoke(app, env=WIDE, args=["format", str(tmp_path)]) + + assert result.exit_code == 1 + output = remove_ansi_color(result.stdout) + assert "could not parse as YAML" in output + # The valid file was still processed despite the earlier bad one. + assert good.read_text(encoding="utf-8").startswith("---\n# yaml-language-server:") + + +def test_format_reports_format_error(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + schema = _write(tmp_path / "dcim.yml", UNFORMATTED) + + def _raise(*_args: object, **_kwargs: object) -> str: + raise FormatError("would change content") + + monkeypatch.setattr(schema_module, "format_schema_text", _raise) + + result = runner.invoke(app, env=WIDE, args=["format", str(schema)]) + + assert result.exit_code == 1 + assert "would change content" in remove_ansi_color(result.stdout) + + +def test_format_skips_non_schema_yaml(tmp_path: Path) -> None: + menu = _write(tmp_path / "menu.yml", "apiVersion: infrahub.app/v1\nkind: Menu\nspec:\n data: []\n") + + result = runner.invoke(app, env=WIDE, args=["format", str(menu)]) + + assert result.exit_code == 0 + assert "0 file(s) reformatted, 0 unchanged" in remove_ansi_color(result.stdout) + + +def test_format_directory_recurses(tmp_path: Path) -> None: + _write(tmp_path / "a.yml", UNFORMATTED) + _write(tmp_path / "b.yml", UNFORMATTED) + + result = runner.invoke(app, env=WIDE, args=["format", str(tmp_path)]) + + assert result.exit_code == 0 + assert "2 file(s) reformatted" in remove_ansi_color(result.stdout) + + +def test_format_opt_in_flags(tmp_path: Path) -> None: + content = """\ +--- +version: "1.0" +nodes: + - namespace: Dcim + name: Device + relationships: + - name: b_rel + peer: DcimB + optional: true + cardinality: many + order_weight: 2000 + - name: a_rel + peer: DcimA + kind: Attribute + cardinality: one +""" + schema = _write(tmp_path / "dcim.yml", content) + + result = runner.invoke( + app, + env=WIDE, + args=["format", str(schema), "--strip-defaults", "--sort-by-order-weight", "--backfill-order-weight"], + ) + + assert result.exit_code == 0 + out = schema.read_text(encoding="utf-8") + rels = yaml.safe_load(out)["nodes"][0]["relationships"] + # backfill filled a_rel (was missing) with 1000, so it sorts before b_rel (2000). + assert [r["name"] for r in rels] == ["a_rel", "b_rel"] + # strip-defaults removed the redundant optional:true / cardinality:many on b_rel. + assert "optional" not in rels[1] + assert "cardinality" not in rels[1] + + +def test_format_leaves_restricted_namespace_untouched(tmp_path: Path) -> None: + content = """\ +--- +version: "1.0" +nodes: + - namespace: Core + name: Special + attributes: + - order_weight: 1000 + kind: Text + name: name +""" + schema = _write(tmp_path / "core.yml", content) + + runner.invoke(app, env=WIDE, args=["format", str(schema)]) + formatted = schema.read_text(encoding="utf-8") + + # The Core node's attribute keys keep their original (scrambled) order. + weight_idx = formatted.index("order_weight: 1000") + name_idx = formatted.index("name: name") + assert weight_idx < name_idx diff --git a/tests/unit/sdk/conftest.py b/tests/unit/sdk/conftest.py index ad5b70fd8..896a45fb1 100644 --- a/tests/unit/sdk/conftest.py +++ b/tests/unit/sdk/conftest.py @@ -154,9 +154,9 @@ async def location_schema() -> NodeSchemaAPI: "namespace": "Builtin", "default_filter": "name__value", "attributes": [ - {"name": "name", "kind": "String", "unique": True}, - {"name": "description", "kind": "String", "optional": True}, - {"name": "type", "kind": "String"}, + {"name": "name", "kind": "Text", "unique": True}, + {"name": "description", "kind": "Text", "optional": True}, + {"name": "type", "kind": "Text"}, ], "relationships": [ { @@ -190,9 +190,9 @@ async def location_schema_with_dropdown() -> NodeSchemaAPI: "namespace": "Builtin", "default_filter": "name__value", "attributes": [ - {"name": "name", "kind": "String", "unique": True}, - {"name": "description", "kind": "String", "optional": True}, - {"name": "type", "kind": "String"}, + {"name": "name", "kind": "Text", "unique": True}, + {"name": "description", "kind": "Text", "optional": True}, + {"name": "type", "kind": "Text"}, { "name": "status", "kind": "Dropdown", @@ -234,9 +234,9 @@ async def schema_with_hfid() -> dict[str, NodeSchemaAPI]: "default_filter": "name__value", "human_friendly_id": ["name__value"], "attributes": [ - {"name": "name", "kind": "String", "unique": True}, - {"name": "description", "kind": "String", "optional": True}, - {"name": "type", "kind": "String"}, + {"name": "name", "kind": "Text", "unique": True}, + {"name": "description", "kind": "Text", "optional": True}, + {"name": "type", "kind": "Text"}, ], "relationships": [ { @@ -266,8 +266,8 @@ async def schema_with_hfid() -> dict[str, NodeSchemaAPI]: "default_filter": "facility_id__value", "human_friendly_id": ["facility_id__value", "location__name__value"], "attributes": [ - {"name": "facility_id", "kind": "String", "unique": True}, - {"name": "description", "kind": "String", "optional": True}, + {"name": "facility_id", "kind": "Text", "unique": True}, + {"name": "description", "kind": "Text", "optional": True}, ], "relationships": [ {"name": "location", "peer": "BuiltinLocation", "cardinality": "one"}, @@ -297,8 +297,8 @@ async def std_group_schema() -> NodeSchemaAPI: "namespace": "Core", "default_filter": "name__value", "attributes": [ - {"name": "name", "kind": "String", "unique": True}, - {"name": "description", "kind": "String", "optional": True}, + {"name": "name", "kind": "Text", "unique": True}, + {"name": "description", "kind": "Text", "optional": True}, ], } return NodeSchema(**data).convert_api() @@ -831,9 +831,9 @@ async def rfile_schema() -> NodeSchemaAPI: "display_labels": ["label__value"], "branch": BranchSupportType.AWARE.value, "attributes": [ - {"name": "name", "kind": "String", "unique": True}, - {"name": "description", "kind": "String", "optional": True}, - {"name": "template_path", "kind": "String"}, + {"name": "name", "kind": "Text", "unique": True}, + {"name": "description", "kind": "Text", "optional": True}, + {"name": "template_path", "kind": "Text"}, ], "relationships": [ { @@ -886,6 +886,21 @@ async def ipaddress_schema() -> NodeSchemaAPI: return NodeSchema(**data).convert_api() +@pytest.fixture +async def bare_ipaddress_schema() -> NodeSchemaAPI: + data = { + "name": "DnsRecord", + "namespace": "Infra", + "default_filter": "address__value", + "display_labels": ["address_value"], + "order_by": ["address_value"], + "attributes": [ + {"name": "address", "kind": "IPAddress"}, + ], + } + return NodeSchema(**data).convert_api() + + @pytest.fixture async def ipnetwork_schema() -> NodeSchemaAPI: data = { @@ -1027,10 +1042,10 @@ async def address_schema() -> NodeSchemaAPI: "display_labels": ["network_value"], "order_by": ["network_value"], "attributes": [ - {"name": "street_number", "kind": "String", "optional": True}, - {"name": "street_name", "kind": "String", "optional": True}, - {"name": "postal_code", "kind": "String", "optional": True}, - {"name": "computed_address", "kind": "String", "optional": True, "read_only": True}, + {"name": "street_number", "kind": "Text", "optional": True}, + {"name": "street_name", "kind": "Text", "optional": True}, + {"name": "postal_code", "kind": "Text", "optional": True}, + {"name": "computed_address", "kind": "Text", "optional": True, "read_only": True}, ], "relationships": [], } @@ -2073,6 +2088,17 @@ async def mock_query_infrahub_version(httpx_mock: HTTPXMock) -> HTTPXMock: return httpx_mock +@pytest.fixture +async def mock_query_infrahub_server_info(httpx_mock: HTTPXMock) -> HTTPXMock: + httpx_mock.add_response( + method="POST", + json={"data": {"InfrahubInfo": {"version": "1.1.0", "deployment_id": "abc123"}}}, + match_headers={"X-Infrahub-Tracker": "query-server-info"}, + is_reusable=True, + ) + return httpx_mock + + @pytest.fixture async def mock_query_infrahub_user(httpx_mock: HTTPXMock) -> HTTPXMock: response_text = (get_fixtures_dir() / "account_profile.json").read_text(encoding="UTF-8") diff --git a/tests/unit/sdk/test_client.py b/tests/unit/sdk/test_client.py index 7b44e0d4b..c20227093 100644 --- a/tests/unit/sdk/test_client.py +++ b/tests/unit/sdk/test_client.py @@ -214,6 +214,19 @@ async def test_method_get_version( assert version == "1.1.0" +@pytest.mark.parametrize("client_type", client_types) +async def test_method_get_server_information( + clients: BothClients, mock_query_infrahub_server_info: HTTPXMock, client_type: str +) -> None: + if client_type == "standard": + server_info = await clients.standard.get_server_information() + else: + server_info = clients.sync.get_server_information() + + assert server_info.version == "1.1.0" + assert server_info.deployment_id == "abc123" + + @pytest.mark.parametrize("client_type", client_types) async def test_method_get_user(clients: BothClients, mock_query_infrahub_user: HTTPXMock, client_type: str) -> None: if client_type == "standard": @@ -280,6 +293,31 @@ async def test_method_all_multiple_pages( assert len(repos) == 5 +@pytest.mark.parametrize("client_type", client_types) +async def test_method_all_pagination_uses_graphql_variables( + httpx_mock: HTTPXMock, + clients: BothClients, + mock_query_repository_page1_2: HTTPXMock, + mock_query_repository_page2_2: HTTPXMock, + client_type: str, +) -> None: + if client_type == "standard": + repos = await clients.standard.all(kind="CoreRepository", populate_store=False) + else: + repos = clients.sync.all(kind="CoreRepository", populate_store=False) + + assert len(repos) == 5 + + payloads = [json.loads(request.content) for request in httpx_mock.get_requests() if request.method == "POST"] + assert len(payloads) == 2 + page1, page2 = payloads + assert page1["query"] == page2["query"] + assert "offset: $offset" in page1["query"] + assert "limit: $limit" in page1["query"] + assert page1["variables"] == {"offset": 0, "limit": 3} + assert page2["variables"] == {"offset": 3, "limit": 3} + + @pytest.mark.parametrize(("client_type", "use_parallel"), batch_client_types) async def test_method_all_batching( clients: BothClients, @@ -755,7 +793,7 @@ async def test_query_name_all( post_requests = [r for r in httpx_mock.get_requests() if r.method == "POST"] assert len(post_requests) == 1 payload = json.loads(post_requests[0].content) - assert "query MyAllQuery {" in payload["query"] + assert "query MyAllQuery ($offset: Int!, $limit: Int!) {" in payload["query"] assert payload["operationName"] == "MyAllQuery" @@ -771,7 +809,7 @@ async def test_query_name_filters( post_requests = [r for r in httpx_mock.get_requests() if r.method == "POST"] assert len(post_requests) == 1 payload = json.loads(post_requests[0].content) - assert "query MyFiltersQuery {" in payload["query"] + assert "query MyFiltersQuery ($offset: Int!, $limit: Int!) {" in payload["query"] assert payload["operationName"] == "MyFiltersQuery" @@ -789,7 +827,7 @@ async def test_query_name_get( post_requests = [r for r in httpx_mock.get_requests() if r.method == "POST"] assert len(post_requests) == 1 payload = json.loads(post_requests[0].content) - assert "query MyGetQuery {" in payload["query"] + assert "query MyGetQuery ($offset: Int!, $limit: Int!) {" in payload["query"] assert payload["operationName"] == "MyGetQuery" @@ -821,7 +859,7 @@ async def test_query_name_for_all_ommitted( post_requests = [r for r in httpx_mock.get_requests() if r.method == "POST"] assert len(post_requests) == 1 payload = json.loads(post_requests[0].content) - assert "query All_CoreRepository {" in payload["query"] + assert "query All_CoreRepository ($offset: Int!, $limit: Int!) {" in payload["query"] assert payload["operationName"] == "All_CoreRepository" @@ -837,7 +875,7 @@ async def test_query_name_for_filters_ommitted( post_requests = [r for r in httpx_mock.get_requests() if r.method == "POST"] assert len(post_requests) == 1 payload = json.loads(post_requests[0].content) - assert "query Filters_CoreRepository {" in payload["query"] + assert "query Filters_CoreRepository ($offset: Int!, $limit: Int!) {" in payload["query"] assert payload["operationName"] == "Filters_CoreRepository" @@ -853,7 +891,7 @@ async def test_query_name_for_get_ommitted( post_requests = [r for r in httpx_mock.get_requests() if r.method == "POST"] assert len(post_requests) == 1 payload = json.loads(post_requests[0].content) - assert "query Get_CoreRepository {" in payload["query"] + assert "query Get_CoreRepository ($offset: Int!, $limit: Int!) {" in payload["query"] assert payload["operationName"] == "Get_CoreRepository" diff --git a/tests/unit/sdk/test_config.py b/tests/unit/sdk/test_config.py index 609e63e3f..527411276 100644 --- a/tests/unit/sdk/test_config.py +++ b/tests/unit/sdk/test_config.py @@ -1,7 +1,11 @@ +from dataclasses import dataclass +from typing import Any + import pytest from pydantic import ValidationError from infrahub_sdk.config import Config +from infrahub_sdk.constants import Priority def test_combine_authentications() -> None: @@ -93,3 +97,73 @@ def test_password_auth_overrides_env_token_when_password_env_var_and_username_ex assert config.password == "testpass" assert config.api_token is None assert config.password_authentication is True + + +def test_invalid_priority_rejected() -> None: + """An unknown configured priority fails at config load; no request is ever issued. + + Construction raises before any client/request exists, so 'no request is issued' + is inherent — the ValidationError is raised while building Config. + """ + # Passing an invalid string is the behaviour under test; pydantic rejects it at load. The field + # statically types Priority | None but coerces strings at runtime, so the dynamic input is passed + # via a dict[str, Any] rather than suppressed with a type-checker ignore. + kwargs: dict[str, Any] = {"address": "http://localhost:8000", "priority": "lowe"} + with pytest.raises(ValidationError, match=r"Input should be 'high', 'medium' or 'low'"): + Config(**kwargs) + + +@dataclass +class PriorityCase: + name: str + value: str | Priority + expected: Priority + + +PRIORITY_CASES = [ + PriorityCase(name="high-upper", value="HIGH", expected=Priority.HIGH), + PriorityCase(name="high-title", value="High", expected=Priority.HIGH), + PriorityCase(name="high-lower", value="high", expected=Priority.HIGH), + PriorityCase(name="high-enum", value=Priority.HIGH, expected=Priority.HIGH), + PriorityCase(name="medium-upper", value="MEDIUM", expected=Priority.MEDIUM), + PriorityCase(name="medium-title", value="Medium", expected=Priority.MEDIUM), + PriorityCase(name="medium-lower", value="medium", expected=Priority.MEDIUM), + PriorityCase(name="medium-enum", value=Priority.MEDIUM, expected=Priority.MEDIUM), + PriorityCase(name="low-upper", value="LOW", expected=Priority.LOW), + PriorityCase(name="low-title", value="Low", expected=Priority.LOW), + PriorityCase(name="low-lower", value="low", expected=Priority.LOW), + PriorityCase(name="low-enum", value=Priority.LOW, expected=Priority.LOW), +] + + +@pytest.mark.parametrize("case", [pytest.param(tc, id=tc.name) for tc in PRIORITY_CASES]) +def test_priority_case_insensitive_acceptance(case: PriorityCase) -> None: + """Valid priority strings are accepted case-insensitively (and enum members pass through).""" + # Case-insensitive string coercion is the behaviour under test; the field statically types Priority, + # so the mixed str/Priority input is passed via a dict[str, Any] rather than suppressed with an ignore. + kwargs: dict[str, Any] = {"address": "http://localhost:8000", "priority": case.value} + config = Config(**kwargs) + assert config.priority is case.expected + + +@pytest.mark.parametrize( + ("env_value", "expected"), + [ + pytest.param("LOW", Priority.LOW, id="low"), + pytest.param("HIGH", Priority.HIGH, id="high"), + pytest.param("MEDIUM", Priority.MEDIUM, id="medium"), + ], +) +def test_priority_from_env_var(monkeypatch: pytest.MonkeyPatch, env_value: str, expected: Priority) -> None: + """The INFRAHUB_PRIORITY env var resolves case-insensitively to a Priority member.""" + monkeypatch.setenv("INFRAHUB_PRIORITY", env_value) + + config = Config(address="http://localhost:8000") + + assert config.priority is expected + + +def test_priority_default_is_none() -> None: + """With no priority configured, the field defaults to None (no header emitted).""" + config = Config(address="http://localhost:8000") + assert config.priority is None diff --git a/tests/unit/sdk/test_hierarchical_nodes.py b/tests/unit/sdk/test_hierarchical_nodes.py index 3165effe0..aa139ba61 100644 --- a/tests/unit/sdk/test_hierarchical_nodes.py +++ b/tests/unit/sdk/test_hierarchical_nodes.py @@ -23,8 +23,8 @@ async def hierarchical_schema() -> NodeSchemaAPI: "namespace": "Infra", "default_filter": "name__value", "attributes": [ - {"name": "name", "kind": "String", "unique": True}, - {"name": "description", "kind": "String", "optional": True}, + {"name": "name", "kind": "Text", "unique": True}, + {"name": "description", "kind": "Text", "optional": True}, ], "relationships": [ { diff --git a/tests/unit/sdk/test_node.py b/tests/unit/sdk/test_node.py index 0a30cb7e3..d8c735637 100644 --- a/tests/unit/sdk/test_node.py +++ b/tests/unit/sdk/test_node.py @@ -1787,6 +1787,23 @@ async def test_create_input_data_with_IPHost_attribute( } +@pytest.mark.parametrize("client_type", client_types) +async def test_create_input_data_with_IPAddress_attribute( + client: InfrahubClient, bare_ipaddress_schema: NodeSchemaAPI, client_type: str +) -> None: + data = {"address": {"value": ipaddress.ip_address("1.1.1.1"), "is_protected": True}} + + if client_type == "standard": + dns_record = InfrahubNode(client=client, schema=bare_ipaddress_schema, data=data) + else: + dns_record = InfrahubNodeSync(client=client, schema=bare_ipaddress_schema, data=data) + + # a bare address is serialized without any prefix + assert dns_record._generate_input_data()["data"] == { + "data": {"address": {"value": "1.1.1.1", "is_protected": True}} + } + + @pytest.mark.parametrize("client_type", client_types) async def test_create_input_data_with_IPNetwork_attribute( client: InfrahubClient, ipnetwork_schema: NodeSchemaAPI, client_type: str @@ -2180,6 +2197,26 @@ async def test_node_IPHost_deserialization( assert ip_address.address.value == ipaddress.ip_interface("1.1.1.1/24") +@pytest.mark.parametrize("client_type", client_types) +async def test_node_IPAddress_deserialization( + client: InfrahubClient, bare_ipaddress_schema: NodeSchemaAPI, client_type: str +) -> None: + data = { + "id": "aaaaaaaaaaaaaa", + "address": { + "value": "1.1.1.1", + "is_protected": True, + }, + } + if client_type == "standard": + dns_record = InfrahubNode(client=client, schema=bare_ipaddress_schema, data=data) + else: + dns_record = InfrahubNodeSync(client=client, schema=bare_ipaddress_schema, data=data) + + # a bare address deserializes to an ip_address object (no prefix) + assert dns_record.address.value == ipaddress.ip_address("1.1.1.1") + + @pytest.mark.parametrize("client_type", client_types) async def test_node_IPNetwork_deserialization( client: InfrahubClient, ipnetwork_schema: NodeSchemaAPI, client_type: str diff --git a/tests/unit/sdk/test_priority.py b/tests/unit/sdk/test_priority.py new file mode 100644 index 000000000..2d03f632a --- /dev/null +++ b/tests/unit/sdk/test_priority.py @@ -0,0 +1,990 @@ +from __future__ import annotations + +import json +from dataclasses import dataclass +from datetime import datetime, timezone +from typing import TYPE_CHECKING + +import pytest + +from infrahub_sdk import Config, InfrahubClient, InfrahubClientSync +from infrahub_sdk.constants import Priority +from infrahub_sdk.context import ContextAccount, RequestContext +from infrahub_sdk.node import InfrahubNode, InfrahubNodeSync +from tests.unit.sdk.conftest import BothClients + +if TYPE_CHECKING: + from pytest_httpx import HTTPXMock + + from infrahub_sdk.schema import NodeSchemaAPI + +pytestmark = pytest.mark.httpx_mock(can_send_already_matched_responses=True) + +client_types = ["standard", "sync"] + + +def _build_clients(priority: Priority) -> BothClients: + return BothClients( + standard=InfrahubClient( + config=Config(address="http://mock", insert_tracker=True, pagination_size=3, priority=priority) + ), + sync=InfrahubClientSync( + config=Config(address="http://mock", insert_tracker=True, pagination_size=3, priority=priority) + ), + ) + + +@pytest.fixture +def low_clients() -> BothClients: + return _build_clients(Priority.LOW) + + +@pytest.mark.parametrize("client_type", client_types) +async def test_priority_header_on_graphql_mutation( + client_type: str, low_clients: BothClients, httpx_mock: HTTPXMock +) -> None: + """A client with a default priority emits X-Priority on a GraphQL mutation.""" + httpx_mock.add_response( + method="POST", + json={"data": {"BuiltinTagCreate": {"ok": True, "object": {"id": "tag-1"}}}}, + match_headers={"X-Priority": "low"}, + ) + + mutation = 'mutation { BuiltinTagCreate(data: {name: {value: "blue"}}) { ok } }' + client = getattr(low_clients, client_type) + if client_type == "standard": + await client.execute_graphql(query=mutation) + else: + client.execute_graphql(query=mutation) + + requests = [r for r in httpx_mock.get_requests() if r.method == "POST"] + assert len(requests) == 1 + assert requests[0].headers["x-priority"] == "low" + + +@pytest.mark.parametrize("client_type", client_types) +async def test_priority_header_on_blob_download( + client_type: str, low_clients: BothClients, httpx_mock: HTTPXMock +) -> None: + """A client with a default priority emits X-Priority on an object-store blob download.""" + httpx_mock.add_response( + method="GET", + text="any content", + match_headers={"X-Priority": "low"}, + ) + + client = getattr(low_clients, client_type) + if client_type == "standard": + content = await client.object_store.get(identifier="aaaaaaaaa") + else: + content = client.object_store.get(identifier="aaaaaaaaa") + + assert content == "any content" + requests = [r for r in httpx_mock.get_requests() if r.method == "GET"] + assert len(requests) == 1 + assert requests[0].headers["x-priority"] == "low" + + +@pytest.mark.parametrize("client_type", client_types) +async def test_priority_header_on_blob_upload( + client_type: str, low_clients: BothClients, httpx_mock: HTTPXMock +) -> None: + """A client with a default priority emits X-Priority on an object-store blob upload.""" + httpx_mock.add_response( + method="POST", + json={"identifier": "xxxxxxxxxx", "checksum": "yyyyyyyyyyyyyy"}, + match_headers={"X-Priority": "low"}, + ) + + client = getattr(low_clients, client_type) + if client_type == "standard": + response = await client.object_store.upload(content="any content") + else: + response = client.object_store.upload(content="any content") + + assert response == {"checksum": "yyyyyyyyyyyyyy", "identifier": "xxxxxxxxxx"} + requests = [r for r in httpx_mock.get_requests() if r.method == "POST"] + assert len(requests) == 1 + assert requests[0].headers["x-priority"] == "low" + + +@pytest.mark.parametrize("client_type", client_types) +async def test_priority_header_on_multipart_upload( + client_type: str, + low_clients: BothClients, + file_object_schema: NodeSchemaAPI, + httpx_mock: HTTPXMock, +) -> None: + """A client with a default priority emits X-Priority on a multipart file upload. + + Confirms the header survives the ``content-type`` pop performed for multipart requests. + """ + httpx_mock.add_response( + method="POST", + json={ + "data": { + "NetworkCircuitContractCreate": { + "ok": True, + "object": { + "id": "new-file-node-123", + "display_label": "contract.pdf", + "file_name": {"value": "contract.pdf"}, + "checksum": {"value": "abc123checksum"}, + "file_size": {"value": 17}, + "file_type": {"value": "application/pdf"}, + "storage_id": {"value": "storage-xyz-789"}, + "contract_start": {"value": "2024-01-01T00:00:00Z"}, + "contract_end": {"value": "2024-12-31T23:59:59Z"}, + }, + } + } + }, + match_headers={"X-Priority": "low"}, + ) + + client = getattr(low_clients, client_type) + if client_type == "standard": + node = InfrahubNode(client=client, schema=file_object_schema, branch="main") + else: + node = InfrahubNodeSync(client=client, schema=file_object_schema, branch="main") + + node._get_attribute("contract_start").value = "2024-01-01T00:00:00Z" + node._get_attribute("contract_end").value = "2024-12-31T23:59:59Z" + node.upload_from_bytes(content=b"Test file content", name="contract.pdf") + + if isinstance(node, InfrahubNode): + await node.save() + else: + node.save() + + requests = [r for r in httpx_mock.get_requests() if r.method == "POST"] + assert len(requests) == 1 + assert requests[0].headers["x-priority"] == "low" + assert requests[0].headers.get("content-type").startswith("multipart/form-data;") + + +@pytest.mark.parametrize("client_type", client_types) +async def test_priority_header_on_batched_requests( + client_type: str, low_clients: BothClients, httpx_mock: HTTPXMock +) -> None: + """Every request issued through a batch carries the client-wide X-Priority header.""" + httpx_mock.add_response( + method="POST", + json={"data": {"InfrahubInfo": {"version": "1.0"}}}, + match_headers={"X-Priority": "low"}, + is_reusable=True, + ) + + query = "query { InfrahubInfo { version }}" + tasks_number = 3 + client = getattr(low_clients, client_type) + + if client_type == "standard": + batch = await client.create_batch() + for _ in range(tasks_number): + batch.add(task=client.execute_graphql, query=query) + async for _, _result in batch.execute(): + pass + else: + batch = client.create_batch() + for _ in range(tasks_number): + batch.add(task=client.execute_graphql, query=query) + for _, _result in batch.execute(): + pass + + requests = [r for r in httpx_mock.get_requests() if r.method == "POST"] + assert len(requests) == tasks_number + assert all(r.headers["x-priority"] == "low" for r in requests) + + +@pytest.mark.parametrize("client_type", client_types) +async def test_no_priority_header_on_blob_download_when_unconfigured( + client_type: str, clients: BothClients, httpx_mock: HTTPXMock +) -> None: + """An unconfigured client emits no X-Priority header on an object-store blob download.""" + httpx_mock.add_response( + method="GET", + text="any content", + ) + + client = getattr(clients, client_type) + if client_type == "standard": + content = await client.object_store.get(identifier="aaaaaaaaa") + else: + content = client.object_store.get(identifier="aaaaaaaaa") + + assert content == "any content" + requests = [r for r in httpx_mock.get_requests() if r.method == "GET"] + assert len(requests) == 1 + assert "x-priority" not in requests[0].headers + + +@pytest.mark.parametrize("client_type", client_types) +async def test_no_priority_header_on_blob_upload_when_unconfigured( + client_type: str, clients: BothClients, httpx_mock: HTTPXMock +) -> None: + """An unconfigured client emits no X-Priority header on an object-store blob upload.""" + httpx_mock.add_response( + method="POST", + json={"identifier": "xxxxxxxxxx", "checksum": "yyyyyyyyyyyyyy"}, + ) + + client = getattr(clients, client_type) + if client_type == "standard": + response = await client.object_store.upload(content="any content") + else: + response = client.object_store.upload(content="any content") + + assert response == {"checksum": "yyyyyyyyyyyyyy", "identifier": "xxxxxxxxxx"} + requests = [r for r in httpx_mock.get_requests() if r.method == "POST"] + assert len(requests) == 1 + assert "x-priority" not in requests[0].headers + + +@pytest.mark.parametrize("client_type", client_types) +async def test_no_priority_header_on_multipart_upload_when_unconfigured( + client_type: str, + clients: BothClients, + file_object_schema: NodeSchemaAPI, + httpx_mock: HTTPXMock, +) -> None: + """An unconfigured client emits no X-Priority header on a multipart file upload.""" + httpx_mock.add_response( + method="POST", + json={ + "data": { + "NetworkCircuitContractCreate": { + "ok": True, + "object": { + "id": "new-file-node-123", + "display_label": "contract.pdf", + "file_name": {"value": "contract.pdf"}, + "checksum": {"value": "abc123checksum"}, + "file_size": {"value": 17}, + "file_type": {"value": "application/pdf"}, + "storage_id": {"value": "storage-xyz-789"}, + "contract_start": {"value": "2024-01-01T00:00:00Z"}, + "contract_end": {"value": "2024-12-31T23:59:59Z"}, + }, + } + } + }, + ) + + client = getattr(clients, client_type) + if client_type == "standard": + node = InfrahubNode(client=client, schema=file_object_schema, branch="main") + else: + node = InfrahubNodeSync(client=client, schema=file_object_schema, branch="main") + + node._get_attribute("contract_start").value = "2024-01-01T00:00:00Z" + node._get_attribute("contract_end").value = "2024-12-31T23:59:59Z" + node.upload_from_bytes(content=b"Test file content", name="contract.pdf") + + if isinstance(node, InfrahubNode): + await node.save() + else: + node.save() + + requests = [r for r in httpx_mock.get_requests() if r.method == "POST"] + assert len(requests) == 1 + assert "x-priority" not in requests[0].headers + assert requests[0].headers.get("content-type").startswith("multipart/form-data;") + + +@pytest.mark.parametrize("client_type", client_types) +async def test_unconfigured_headers_unchanged_versus_baseline( + client_type: str, clients: BothClients, httpx_mock: HTTPXMock +) -> None: + """With no priority and no per-request arg, the SDK-set outgoing headers are unchanged. + + Only the absence of X-Priority matters; the request still carries the baseline SDK + headers it always had (``content-type`` and, since ``insert_tracker`` is set, the + ``X-Infrahub-Tracker`` header). Transport-injected headers (host, user-agent, etc.) + are intentionally not asserted. + """ + httpx_mock.add_response( + method="POST", + json={"data": {"InfrahubInfo": {"version": "1.0"}}}, + ) + + query = "query { InfrahubInfo { version }}" + tracker = "test-priority-baseline" + client = getattr(clients, client_type) + if client_type == "standard": + await client.execute_graphql(query=query, tracker=tracker) + else: + client.execute_graphql(query=query, tracker=tracker) + + requests = [r for r in httpx_mock.get_requests() if r.method == "POST"] + assert len(requests) == 1 + request = requests[0] + assert "x-priority" not in request.headers + assert request.headers["content-type"].startswith("application/json") + assert request.headers["x-infrahub-tracker"] == tracker + + +# --------------------------------------------------------------------------- +# Per-request override +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize("client_type", client_types) +async def test_override_on_no_default_client_then_no_leak( + client_type: str, clients: BothClients, httpx_mock: HTTPXMock +) -> None: + """On a no-default client, priority=HIGH emits the header; the next un-annotated call emits none.""" + httpx_mock.add_response( + method="POST", + json={"data": {"InfrahubInfo": {"version": "1.0"}}}, + is_reusable=True, + ) + + query = "query { InfrahubInfo { version }}" + client = getattr(clients, client_type) + if client_type == "standard": + await client.execute_graphql(query=query, priority=Priority.HIGH) + await client.execute_graphql(query=query) + else: + client.execute_graphql(query=query, priority=Priority.HIGH) + client.execute_graphql(query=query) + + requests = [r for r in httpx_mock.get_requests() if r.method == "POST"] + assert len(requests) == 2 + assert requests[0].headers["x-priority"] == "high" + assert "x-priority" not in requests[1].headers + + +@pytest.mark.parametrize("client_type", client_types) +async def test_override_beats_default_then_reverts( + client_type: str, low_clients: BothClients, httpx_mock: HTTPXMock +) -> None: + """A per-request HIGH overrides a LOW default for one call; the next call reverts to LOW.""" + httpx_mock.add_response( + method="POST", + json={"data": {"InfrahubInfo": {"version": "1.0"}}}, + is_reusable=True, + ) + + query = "query { InfrahubInfo { version }}" + client = getattr(low_clients, client_type) + if client_type == "standard": + await client.execute_graphql(query=query, priority=Priority.HIGH) + await client.execute_graphql(query=query) + else: + client.execute_graphql(query=query, priority=Priority.HIGH) + client.execute_graphql(query=query) + + requests = [r for r in httpx_mock.get_requests() if r.method == "POST"] + assert len(requests) == 2 + assert requests[0].headers["x-priority"] == "high" + assert requests[1].headers["x-priority"] == "low" + + +@pytest.mark.parametrize("client_type", client_types) +async def test_override_on_get( + client_type: str, clients: BothClients, mock_schema_query_01: HTTPXMock, httpx_mock: HTTPXMock +) -> None: + """A per-request priority on get() reaches the underlying GraphQL request.""" + response = { + "data": { + "CoreRepository": { + "edges": [ + { + "node": { + "__typename": "CoreRepository", + "id": "bfae43e8-5ebb-456c-a946-bf64e930710a", + "name": {"value": "infrahub-demo-core"}, + "location": {"value": "git@github.com:opsmill/infrahub-demo-core.git"}, + "commit": {"value": "bbbbbbbbbbbbbbbbbbbb"}, + } + } + ] + } + } + } + httpx_mock.add_response( + method="POST", + json=response, + match_headers={"X-Priority": "high"}, + is_reusable=True, + ) + + node_id = "bfae43e8-5ebb-456c-a946-bf64e930710a" + client = getattr(clients, client_type) + if client_type == "standard": + await client.get(kind="CoreRepository", id=node_id, priority=Priority.HIGH) + else: + client.get(kind="CoreRepository", id=node_id, priority=Priority.HIGH) + + query_requests = [ + r + for r in httpx_mock.get_requests() + if r.method == "POST" and r.headers.get("x-infrahub-tracker") == "query-corerepository-page1" + ] + assert len(query_requests) == 1 + assert query_requests[0].headers["x-priority"] == "high" + + +@pytest.mark.parametrize("client_type", client_types) +async def test_override_on_all_carries_on_every_page( + client_type: str, + clients: BothClients, + mock_query_repository_page1_2: HTTPXMock, + mock_query_repository_page2_2: HTTPXMock, + httpx_mock: HTTPXMock, +) -> None: + """The override is forwarded on every page request of a paginated all().""" + client = getattr(clients, client_type) + if client_type == "standard": + repos = await client.all(kind="CoreRepository", priority=Priority.HIGH) + else: + repos = client.all(kind="CoreRepository", priority=Priority.HIGH) + assert len(repos) == 5 + + page_requests = [ + r + for r in httpx_mock.get_requests() + if r.method == "POST" and (r.headers.get("x-infrahub-tracker") or "").startswith("query-corerepository-page") + ] + assert len(page_requests) == 2 + assert all(r.headers["x-priority"] == "high" for r in page_requests) + + +@pytest.mark.parametrize("client_type", client_types) +async def test_override_on_save_create_path( + client_type: str, clients: BothClients, location_schema: NodeSchemaAPI, httpx_mock: HTTPXMock +) -> None: + """A per-request priority on node.save() reaches the create mutation.""" + httpx_mock.add_response( + method="POST", + json={ + "data": {"BuiltinLocationCreate": {"ok": True, "object": {"id": "17aec828-9814-ce00-3f20-1a053670f1c8"}}} + }, + is_reusable=True, + ) + + client = getattr(clients, client_type) + data = {"name": {"value": "JFK1"}, "type": {"value": "SITE"}} + if client_type == "standard": + node = InfrahubNode(client=client, schema=location_schema, data=data) + await node.save(priority=Priority.HIGH) + else: + node = InfrahubNodeSync(client=client, schema=location_schema, data=data) + node.save(priority=Priority.HIGH) + + create_requests = [ + r + for r in httpx_mock.get_requests() + if r.method == "POST" and r.headers.get("x-infrahub-tracker") == "mutation-builtinlocation-create" + ] + assert len(create_requests) == 1 + assert create_requests[0].headers["x-priority"] == "high" + + +@pytest.mark.parametrize("client_type", client_types) +async def test_override_on_diff_method(client_type: str, clients: BothClients, httpx_mock: HTTPXMock) -> None: + """A per-request priority on a diff method reaches the GraphQL request.""" + httpx_mock.add_response( + method="POST", + json={"data": {"DiffUpdate": {"ok": True}}}, + match_headers={"X-Priority": "high"}, + ) + + from_time = datetime(2024, 1, 1, tzinfo=timezone.utc) + to_time = datetime(2024, 1, 2, tzinfo=timezone.utc) + client = getattr(clients, client_type) + if client_type == "standard": + await client.create_diff( + branch="main", name="test-diff", from_time=from_time, to_time=to_time, priority=Priority.HIGH + ) + else: + client.create_diff( + branch="main", name="test-diff", from_time=from_time, to_time=to_time, priority=Priority.HIGH + ) + + requests = [r for r in httpx_mock.get_requests() if r.method == "POST"] + assert len(requests) == 1 + assert requests[0].headers["x-priority"] == "high" + + +@pytest.mark.parametrize("client_type", client_types) +async def test_override_on_get_diff_summary(client_type: str, clients: BothClients, httpx_mock: HTTPXMock) -> None: + """A per-request priority on get_diff_summary() reaches the GraphQL request.""" + httpx_mock.add_response( + method="POST", + json={"data": {"DiffTree": {"nodes": []}}}, + match_headers={"X-Priority": "high"}, + ) + + client = getattr(clients, client_type) + if client_type == "standard": + await client.get_diff_summary(branch="main", priority=Priority.HIGH) + else: + client.get_diff_summary(branch="main", priority=Priority.HIGH) + + requests = [r for r in httpx_mock.get_requests() if r.method == "POST"] + assert len(requests) == 1 + assert requests[0].headers["x-priority"] == "high" + + +@pytest.mark.parametrize("client_type", client_types) +async def test_override_on_get_diff_tree(client_type: str, clients: BothClients, httpx_mock: HTTPXMock) -> None: + """A per-request priority on get_diff_tree() reaches the GraphQL request.""" + httpx_mock.add_response( + method="POST", + json={"data": {"DiffTree": None}}, + match_headers={"X-Priority": "high"}, + ) + + client = getattr(clients, client_type) + if client_type == "standard": + await client.get_diff_tree(branch="main", priority=Priority.HIGH) + else: + client.get_diff_tree(branch="main", priority=Priority.HIGH) + + requests = [r for r in httpx_mock.get_requests() if r.method == "POST"] + assert len(requests) == 1 + assert requests[0].headers["x-priority"] == "high" + + +@pytest.mark.parametrize("client_type", client_types) +async def test_override_on_save_update_path( + client_type: str, clients: BothClients, location_schema: NodeSchemaAPI, httpx_mock: HTTPXMock +) -> None: + """A per-request priority on node.save() reaches the update mutation for an existing node.""" + httpx_mock.add_response( + method="POST", + json={ + "data": {"BuiltinLocationUpdate": {"ok": True, "object": {"id": "17aec828-9814-ce00-3f20-1a053670f1c8"}}} + }, + is_reusable=True, + ) + + data = {"id": "17aec828-9814-ce00-3f20-1a053670f1c8", "name": {"value": "JFK1"}, "type": {"value": "SITE"}} + client = getattr(clients, client_type) + if client_type == "standard": + node = InfrahubNode(client=client, schema=location_schema, data=data) + node._get_attribute("name").value = "JFK2" + await node.save(priority=Priority.HIGH) + else: + node = InfrahubNodeSync(client=client, schema=location_schema, data=data) + node._get_attribute("name").value = "JFK2" + node.save(priority=Priority.HIGH) + + update_requests = [ + r + for r in httpx_mock.get_requests() + if r.method == "POST" and r.headers.get("x-infrahub-tracker") == "mutation-builtinlocation-update" + ] + assert len(update_requests) == 1 + assert update_requests[0].headers["x-priority"] == "high" + + +@pytest.mark.parametrize("client_type", client_types) +async def test_override_on_node_delete( + client_type: str, clients: BothClients, location_schema: NodeSchemaAPI, httpx_mock: HTTPXMock +) -> None: + """A per-request priority on node.delete() reaches the delete mutation.""" + httpx_mock.add_response( + method="POST", + json={"data": {"BuiltinLocationDelete": {"ok": True}}}, + is_reusable=True, + ) + + data = {"id": "17aec828-9814-ce00-3f20-1a053670f1c8", "name": {"value": "JFK1"}, "type": {"value": "SITE"}} + client = getattr(clients, client_type) + if client_type == "standard": + node = InfrahubNode(client=client, schema=location_schema, data=data) + await node.delete(priority=Priority.HIGH) + else: + node = InfrahubNodeSync(client=client, schema=location_schema, data=data) + node.delete(priority=Priority.HIGH) + + delete_requests = [ + r + for r in httpx_mock.get_requests() + if r.method == "POST" and r.headers.get("x-infrahub-tracker") == "mutation-builtinlocation-delete" + ] + assert len(delete_requests) == 1 + assert delete_requests[0].headers["x-priority"] == "high" + + +@pytest.mark.parametrize("client_type", client_types) +async def test_override_on_multipart_upload( + client_type: str, + clients: BothClients, + file_object_schema: NodeSchemaAPI, + httpx_mock: HTTPXMock, +) -> None: + """A per-request priority survives the multipart content-type pop on a file upload.""" + httpx_mock.add_response( + method="POST", + json={ + "data": { + "NetworkCircuitContractCreate": { + "ok": True, + "object": { + "id": "new-file-node-123", + "display_label": "contract.pdf", + "file_name": {"value": "contract.pdf"}, + "checksum": {"value": "abc123checksum"}, + "file_size": {"value": 17}, + "file_type": {"value": "application/pdf"}, + "storage_id": {"value": "storage-xyz-789"}, + "contract_start": {"value": "2024-01-01T00:00:00Z"}, + "contract_end": {"value": "2024-12-31T23:59:59Z"}, + }, + } + } + }, + match_headers={"X-Priority": "high"}, + ) + + client = getattr(clients, client_type) + if client_type == "standard": + node = InfrahubNode(client=client, schema=file_object_schema, branch="main") + else: + node = InfrahubNodeSync(client=client, schema=file_object_schema, branch="main") + + node._get_attribute("contract_start").value = "2024-01-01T00:00:00Z" + node._get_attribute("contract_end").value = "2024-12-31T23:59:59Z" + node.upload_from_bytes(content=b"Test file content", name="contract.pdf") + + if isinstance(node, InfrahubNode): + await node.save(priority=Priority.HIGH) + else: + node.save(priority=Priority.HIGH) + + requests = [r for r in httpx_mock.get_requests() if r.method == "POST"] + assert len(requests) == 1 + assert requests[0].headers["x-priority"] == "high" + assert requests[0].headers.get("content-type").startswith("multipart/form-data;") + + +# --------------------------------------------------------------------------- +# Async / sync parity +# --------------------------------------------------------------------------- + + +@dataclass +class ResolutionCase: + """One row of the priority resolution truth table. + + ``expected`` is the emitted ``X-Priority`` header value, or ``None`` when no + header should be present. + """ + + name: str + client_default: Priority | None + per_request: Priority | None + expected: str | None + + +# The full priority resolution truth table. Each row must resolve +# identically on both the async and sync clients. +RESOLUTION_TRUTH_TABLE = [ + ResolutionCase(name="no-default-no-override", client_default=None, per_request=None, expected=None), + ResolutionCase(name="no-default-override-high", client_default=None, per_request=Priority.HIGH, expected="high"), + ResolutionCase( + name="no-default-override-medium", client_default=None, per_request=Priority.MEDIUM, expected="medium" + ), + ResolutionCase(name="low-default-no-override", client_default=Priority.LOW, per_request=None, expected="low"), + ResolutionCase( + name="low-default-override-high", client_default=Priority.LOW, per_request=Priority.HIGH, expected="high" + ), + ResolutionCase( + name="low-default-override-medium", client_default=Priority.LOW, per_request=Priority.MEDIUM, expected="medium" + ), + ResolutionCase( + name="medium-default-no-override", client_default=Priority.MEDIUM, per_request=None, expected="medium" + ), + ResolutionCase( + name="high-default-override-low", client_default=Priority.HIGH, per_request=Priority.LOW, expected="low" + ), +] + + +def _client_with_default(client_type: str, default: Priority | None) -> InfrahubClient | InfrahubClientSync: + config = Config(address="http://mock", insert_tracker=True, pagination_size=3, priority=default) + if client_type == "standard": + return InfrahubClient(config=config) + return InfrahubClientSync(config=config) + + +@pytest.mark.parametrize("client_type", client_types) +@pytest.mark.parametrize("case", [pytest.param(tc, id=tc.name) for tc in RESOLUTION_TRUTH_TABLE]) +async def test_resolution_truth_table_parity(case: ResolutionCase, client_type: str, httpx_mock: HTTPXMock) -> None: + """Each (client_default x per_request) combination emits the same header on both clients. + + Runs every row of the priority resolution truth table against both the async and sync + clients, asserting identical emitted headers. + """ + httpx_mock.add_response( + method="POST", + json={"data": {"InfrahubInfo": {"version": "1.0"}}}, + ) + + query = "query { InfrahubInfo { version }}" + client = _client_with_default(client_type, case.client_default) + kwargs = {} if case.per_request is None else {"priority": case.per_request} + + if client_type == "standard": + await client.execute_graphql(query=query, **kwargs) # type: ignore[misc] + else: + client.execute_graphql(query=query, **kwargs) # type: ignore[union-attr] + + requests = [r for r in httpx_mock.get_requests() if r.method == "POST"] + assert len(requests) == 1 + if case.expected is None: + assert "x-priority" not in requests[0].headers + else: + assert requests[0].headers["x-priority"] == case.expected + + +@pytest.mark.parametrize("client_type", client_types) +async def test_count_carries_priority( + client_type: str, clients: BothClients, mock_schema_query_01: HTTPXMock, httpx_mock: HTTPXMock +) -> None: + """A per-request priority on count() reaches the count query.""" + httpx_mock.add_response( + method="POST", + json={"data": {"CoreRepository": {"count": 5}}}, + match_headers={"X-Priority": "high"}, + is_reusable=True, + ) + + client = getattr(clients, client_type) + if client_type == "standard": + result = await client.count(kind="CoreRepository", priority=Priority.HIGH) + else: + result = client.count(kind="CoreRepository", priority=Priority.HIGH) + assert result == 5 + + +@pytest.mark.parametrize("client_type", client_types) +async def test_override_on_all_parallel_count_query( + client_type: str, + clients: BothClients, + mock_query_repository_page1_2: HTTPXMock, + mock_query_repository_page2_2: HTTPXMock, + httpx_mock: HTTPXMock, +) -> None: + """In parallel mode, the preliminary count query carries the override too, not just the pages.""" + # Registered after the tracker-matched page fixtures so the untracked count query falls through here. + httpx_mock.add_response( + method="POST", + json={"data": {"CoreRepository": {"count": 5}}}, + is_reusable=True, + ) + + client = getattr(clients, client_type) + if client_type == "standard": + await client.all(kind="CoreRepository", parallel=True, priority=Priority.HIGH) + else: + client.all(kind="CoreRepository", parallel=True, priority=Priority.HIGH) + + count_requests = [ + r for r in httpx_mock.get_requests() if r.method == "POST" and b"Count_CoreRepository" in r.read() + ] + assert count_requests + assert all(r.headers["x-priority"] == "high" for r in count_requests) + + +@pytest.mark.parametrize("client_type", client_types) +async def test_related_node_fetch_forwards_priority( + client_type: str, + clients: BothClients, + mock_schema_query_01: HTTPXMock, + location_schema: NodeSchemaAPI, + location_data01: dict, + tag_schema: NodeSchemaAPI, + tag_blue_data: dict, + httpx_mock: HTTPXMock, +) -> None: + """A per-request priority passed to RelatedNode.fetch() reaches the peer query. + + This is the path a node create/update with a resource-pool relationship uses for its + follow-up peer fetch, so the whole operation carries a single consistent priority. + """ + httpx_mock.add_response( + method="POST", + json={"data": {"BuiltinTag": {"count": 1, "edges": [tag_blue_data]}}}, + match_headers={"X-Priority": "high"}, + is_reusable=True, + ) + + client = getattr(clients, client_type) + if client_type == "standard": + node = InfrahubNode(client=client, schema=location_schema, data=location_data01) + await node.primary_tag.fetch(priority=Priority.HIGH) # type: ignore[attr-defined] + else: + node = InfrahubNodeSync(client=client, schema=location_schema, data=location_data01) + node.primary_tag.fetch(priority=Priority.HIGH) # type: ignore[attr-defined] + + tag_requests = [ + r + for r in httpx_mock.get_requests() + if r.method == "POST" and (r.headers.get("x-infrahub-tracker") or "").startswith("query-builtintag") + ] + assert tag_requests + assert all(r.headers["x-priority"] == "high" for r in tag_requests) + + +# --------------------------------------------------------------------------- +# request_context.priority — priority carried on the client's RequestContext +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize("client_type", client_types) +async def test_request_context_priority_on_graphql( + client_type: str, clients: BothClients, httpx_mock: HTTPXMock +) -> None: + """A priority set on the client's request_context emits X-Priority on a GraphQL request.""" + httpx_mock.add_response( + method="POST", + json={"data": {"InfrahubInfo": {"version": "1.0"}}}, + match_headers={"X-Priority": "low"}, + ) + + client = getattr(clients, client_type) + client.request_context = RequestContext(priority=Priority.LOW) + query = "query { InfrahubInfo { version }}" + if client_type == "standard": + await client.execute_graphql(query=query) + else: + client.execute_graphql(query=query) + + requests = [r for r in httpx_mock.get_requests() if r.method == "POST"] + assert len(requests) == 1 + assert requests[0].headers["x-priority"] == "low" + + +@pytest.mark.parametrize("client_type", client_types) +async def test_request_context_priority_on_object_store_blob( + client_type: str, clients: BothClients, httpx_mock: HTTPXMock +) -> None: + """Blob requests (which take no per-call priority kwarg) inherit request_context.priority.""" + httpx_mock.add_response( + method="GET", + text="any content", + match_headers={"X-Priority": "low"}, + ) + + client = getattr(clients, client_type) + client.request_context = RequestContext(priority=Priority.LOW) + if client_type == "standard": + await client.object_store.get(identifier="aaaaaaaaa") + else: + client.object_store.get(identifier="aaaaaaaaa") + + requests = [r for r in httpx_mock.get_requests() if r.method == "GET"] + assert len(requests) == 1 + assert requests[0].headers["x-priority"] == "low" + + +@pytest.mark.parametrize("client_type", client_types) +async def test_request_context_priority_not_in_mutation_body( + client_type: str, clients: BothClients, location_schema: NodeSchemaAPI, httpx_mock: HTTPXMock +) -> None: + """Priority rides the header only — the account still reaches the mutation body, priority never does.""" + httpx_mock.add_response( + method="POST", + json={ + "data": {"BuiltinLocationCreate": {"ok": True, "object": {"id": "17aec828-9814-ce00-3f20-1a053670f1c8"}}} + }, + is_reusable=True, + ) + + client = getattr(clients, client_type) + client.request_context = RequestContext(account=ContextAccount(id="acc-1"), priority=Priority.LOW) + data = {"name": {"value": "JFK1"}, "type": {"value": "SITE"}} + if client_type == "standard": + node = InfrahubNode(client=client, schema=location_schema, data=data) + await node.save() + else: + node = InfrahubNodeSync(client=client, schema=location_schema, data=data) + node.save() + + create_requests = [ + r + for r in httpx_mock.get_requests() + if r.method == "POST" and r.headers.get("x-infrahub-tracker") == "mutation-builtinlocation-create" + ] + assert len(create_requests) == 1 + assert create_requests[0].headers["x-priority"] == "low" + + # the account context is rendered into the mutation, but priority must never leak into the body + query = json.loads(create_requests[0].content)["query"] + assert "acc-1" in query + assert "priority" not in query + + +@dataclass +class RequestContextResolutionCase: + """One row of the request_context priority resolution truth table.""" + + name: str + client_default: Priority | None + request_context_priority: Priority | None + per_request: Priority | None + expected: str | None + + +# Precedence: per-call kwarg > request_context.priority > client default (Config.priority) > none. +REQUEST_CONTEXT_TRUTH_TABLE = [ + RequestContextResolutionCase( + name="rc-only", client_default=None, request_context_priority=Priority.LOW, per_request=None, expected="low" + ), + RequestContextResolutionCase( + name="rc-none-no-default", client_default=None, request_context_priority=None, per_request=None, expected=None + ), + RequestContextResolutionCase( + name="rc-beats-default", + client_default=Priority.MEDIUM, + request_context_priority=Priority.LOW, + per_request=None, + expected="low", + ), + RequestContextResolutionCase( + name="kwarg-beats-rc", + client_default=None, + request_context_priority=Priority.LOW, + per_request=Priority.HIGH, + expected="high", + ), + RequestContextResolutionCase( + name="default-when-rc-none", + client_default=Priority.LOW, + request_context_priority=None, + per_request=None, + expected="low", + ), +] + + +@pytest.mark.parametrize("client_type", client_types) +@pytest.mark.parametrize("case", [pytest.param(tc, id=tc.name) for tc in REQUEST_CONTEXT_TRUTH_TABLE]) +async def test_request_context_precedence_parity( + case: RequestContextResolutionCase, client_type: str, httpx_mock: HTTPXMock +) -> None: + """Kwarg > request_context.priority > Config.priority default > none, identically on both clients.""" + httpx_mock.add_response(method="POST", json={"data": {"InfrahubInfo": {"version": "1.0"}}}) + + client = _client_with_default(client_type, case.client_default) + if case.request_context_priority is not None: + client.request_context = RequestContext(priority=case.request_context_priority) + query = "query { InfrahubInfo { version }}" + kwargs = {} if case.per_request is None else {"priority": case.per_request} + + if client_type == "standard": + await client.execute_graphql(query=query, **kwargs) # type: ignore[misc] + else: + client.execute_graphql(query=query, **kwargs) # type: ignore[union-attr] + + requests = [r for r in httpx_mock.get_requests() if r.method == "POST"] + assert len(requests) == 1 + if case.expected is None: + assert "x-priority" not in requests[0].headers + else: + assert requests[0].headers["x-priority"] == case.expected diff --git a/tests/unit/sdk/test_rate_limit_retry.py b/tests/unit/sdk/test_rate_limit_retry.py new file mode 100644 index 000000000..8306afb15 --- /dev/null +++ b/tests/unit/sdk/test_rate_limit_retry.py @@ -0,0 +1,770 @@ +"""Client-level tests for transparent HTTP 429 retry on the async and sync clients. + +Covers transparent 429->200 retry, honouring ``Retry-After``, ``RateLimitError`` on +exhaustion, the disabled path, async/sync parity, all-paths coverage (regular request, +multipart, streaming init), and the multipart body re-read regression. +""" + +from __future__ import annotations + +import io +import logging +import re +from collections.abc import Callable +from dataclasses import dataclass +from datetime import datetime, timedelta, timezone +from email.utils import format_datetime +from typing import TYPE_CHECKING, Any + +import httpx +import pytest + +from infrahub_sdk import InfrahubClient, InfrahubClientSync +from infrahub_sdk import client as client_module +from infrahub_sdk.config import Config +from infrahub_sdk.exceptions import RateLimitError +from infrahub_sdk.rate_limit import RateLimitRetryHandler +from infrahub_sdk.types import HTTPMethod + +if TYPE_CHECKING: + from pytest_httpx import HTTPXMock + +CLIENT_TYPES = ["standard", "sync"] + + +class ScriptedRequester: + """A pluggable ``requester``/``sync_requester`` that replays a scripted response sequence. + + Each invocation returns the next pre-built ``httpx.Response`` and increments ``call_count``, + letting a test assert exactly how many HTTP sends the retry driver performed. + """ + + def __init__(self, responses: list[httpx.Response]) -> None: + self._responses = responses + self.call_count = 0 + + def _next(self) -> httpx.Response: + response = self._responses[self.call_count] + self.call_count += 1 + return response + + def sync_request( + self, + url: str, + method: HTTPMethod, + headers: dict[str, Any], + timeout: int, + payload: dict | None = None, + ) -> httpx.Response: + return self._next() + + async def async_request( + self, + url: str, + method: HTTPMethod, + headers: dict[str, Any], + timeout: int, + payload: dict | None = None, + ) -> httpx.Response: + return self._next() + + +def _patch_driver_sleep(monkeypatch: pytest.MonkeyPatch) -> list[float]: + """Replace the driver's async/sync sleep with no-op recorders so tests never really wait. + + Returns the list that captures every recorded delay, in call order. + """ + recorded: list[float] = [] + + async def fake_async_sleep(delay: float) -> None: + recorded.append(delay) + + def fake_sync_sleep(delay: float) -> None: + recorded.append(delay) + + monkeypatch.setattr(client_module.asyncio, "sleep", fake_async_sleep) + monkeypatch.setattr(client_module.time, "sleep", fake_sync_sleep) + return recorded + + +async def _send_request( + client_type: str, + requester: ScriptedRequester, + url: str = "http://mock/graphql/main", + max_retries: int | None = None, +) -> httpx.Response: + """Drive the real ``_request`` path on the selected client with the scripted requester. + + ``max_retries`` overrides ``rate_limit_max_retries`` on the client's ``Config`` when set. + """ + overrides: dict[str, Any] = {} if max_retries is None else {"rate_limit_max_retries": max_retries} + if client_type == "standard": + config = Config(address="http://mock", requester=requester.async_request, **overrides) + client = InfrahubClient(config=config) + return await client._request(url=url, method=HTTPMethod.POST, headers={}, timeout=10, payload={}) + + config = Config(address="http://mock", sync_requester=requester.sync_request, **overrides) + client_sync = InfrahubClientSync(config=config) + return client_sync._request(url=url, method=HTTPMethod.POST, headers={}, timeout=10, payload={}) + + +@pytest.mark.parametrize("client_type", CLIENT_TYPES) +async def test_request_retries_429_then_succeeds(client_type: str, monkeypatch: pytest.MonkeyPatch) -> None: + """A 429 followed by a 200 is retried transparently: the 200 is returned after two sends.""" + recorded_sleeps = _patch_driver_sleep(monkeypatch) + + success_payload = {"data": {"result": "success"}} + requester = ScriptedRequester( + [ + httpx.Response(status_code=429), + httpx.Response(status_code=200, json=success_payload), + ] + ) + + response = await _send_request(client_type=client_type, requester=requester) + + assert response.status_code == 200 + assert response.json() == success_payload + assert requester.call_count == 2 + assert len(recorded_sleeps) == 1 + + +@pytest.mark.parametrize("status_code", [200, 500]) +@pytest.mark.parametrize("client_type", CLIENT_TYPES) +async def test_request_passes_non_429_through_untouched( + client_type: str, status_code: int, monkeypatch: pytest.MonkeyPatch +) -> None: + """Non-429 responses (success or error) are returned on the first send with no retry or wait.""" + recorded_sleeps = _patch_driver_sleep(monkeypatch) + + requester = ScriptedRequester([httpx.Response(status_code=status_code, json={"data": None})]) + + response = await _send_request(client_type=client_type, requester=requester) + + assert response.status_code == status_code + assert requester.call_count == 1 + assert recorded_sleeps == [] + + +@dataclass +class RetryAfterCase: + """A ``Retry-After`` header form and the inclusive wait window the driver must sleep for. + + ``build_header`` is evaluated at test time so date-relative forms are computed against the + current clock (the driver parses them against ``datetime.now``). + """ + + name: str + build_header: Callable[[], str] + lower: float + upper: float + + +# Default ``rate_limit_backoff_max`` is 60.0s, so a ``Retry-After`` above it clamps to 60.0. +RETRY_AFTER_CASES = [ + RetryAfterCase(name="delta-seconds", build_header=lambda: "5", lower=5.0, upper=5.0), + RetryAfterCase( + name="http-date", + build_header=lambda: format_datetime(datetime.now(timezone.utc) + timedelta(seconds=30), usegmt=True), + # A few seconds elapse between building the header and the driver parsing it, so the + # honoured wait lands just under the 30s interval. + lower=25.0, + upper=30.1, + ), + RetryAfterCase(name="zero-seconds", build_header=lambda: "0", lower=0.0, upper=0.0), + RetryAfterCase( + name="past-date", + build_header=lambda: format_datetime(datetime.now(timezone.utc) - timedelta(seconds=30), usegmt=True), + lower=0.0, + upper=0.0, + ), + RetryAfterCase(name="above-max-clamped", build_header=lambda: "120", lower=60.0, upper=60.0), +] + + +@pytest.mark.parametrize("case", [pytest.param(tc, id=tc.name) for tc in RETRY_AFTER_CASES]) +@pytest.mark.parametrize("client_type", CLIENT_TYPES) +async def test_request_honours_retry_after( + client_type: str, case: RetryAfterCase, monkeypatch: pytest.MonkeyPatch +) -> None: + """A parseable ``Retry-After`` on the 429 dictates the wait (clamped to the backoff ceiling).""" + recorded_sleeps = _patch_driver_sleep(monkeypatch) + + success_payload = {"data": {"result": "success"}} + requester = ScriptedRequester( + [ + httpx.Response(status_code=429, headers={"Retry-After": case.build_header()}), + httpx.Response(status_code=200, json=success_payload), + ] + ) + + response = await _send_request(client_type=client_type, requester=requester) + + assert response.status_code == 200 + assert response.json() == success_payload + assert requester.call_count == 2 + assert len(recorded_sleeps) == 1 + assert case.lower <= recorded_sleeps[0] <= case.upper + + +@pytest.mark.parametrize("client_type", CLIENT_TYPES) +async def test_request_malformed_retry_after_falls_back_to_backoff( + client_type: str, monkeypatch: pytest.MonkeyPatch +) -> None: + """A malformed ``Retry-After`` is ignored: the driver still retries using computed backoff.""" + recorded_sleeps = _patch_driver_sleep(monkeypatch) + + success_payload = {"data": {"result": "success"}} + requester = ScriptedRequester( + [ + httpx.Response(status_code=429, headers={"Retry-After": "not-a-real-header"}), + httpx.Response(status_code=200, json=success_payload), + ] + ) + + response = await _send_request(client_type=client_type, requester=requester) + + assert response.status_code == 200 + # The retry still happened despite the unparseable header. + assert requester.call_count == 2 + assert len(recorded_sleeps) == 1 + + # The wait came from jittered exponential backoff for the first retry (attempt=0), not the + # header, so it lands within ``[0, compute_backoff(0)]`` of a handler built from Config defaults. + defaults = Config(address="http://mock") + handler = RateLimitRetryHandler( + max_retries=defaults.rate_limit_max_retries, + backoff_base=defaults.rate_limit_backoff_base, + backoff_max=defaults.rate_limit_backoff_max, + ) + ceiling = handler.compute_backoff(attempt=0) + assert 0.0 <= recorded_sleeps[0] <= ceiling + + +# The driver logs each retry through ``logging.getLogger("infrahub_sdk")`` (client ``self.log``). +_RETRY_LOG_LOGGER = "infrahub_sdk" +# Matches the driver's WARNING format: "Rate limited (HTTP 429) on , retry in s". +_RETRY_LOG_PATTERN = re.compile(r"retry (?P\d+) in (?P[\d.]+)s") + + +@pytest.mark.parametrize("client_type", CLIENT_TYPES) +async def test_request_exhausts_retries_and_raises_rate_limit_error( + client_type: str, monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture +) -> None: + """Persistent 429 exhausts the budget: exactly ``max_retries + 1`` sends, then one ``RateLimitError``. + + The raised error carries ``url``/``attempts``/``retry_after`` and chains the terminal + ``httpx.HTTPStatusError`` as ``__cause__``; one WARNING per retry is logged with the url, + the attempt number, and the honoured delay. + """ + recorded_sleeps = _patch_driver_sleep(monkeypatch) + + max_retries = 3 + url = "http://mock/graphql/main" + # Every send returns a 429 carrying ``Retry-After`` so ``err.retry_after`` is populated. A + # ``request`` is attached (as a real transport always does) so the driver's terminal + # ``raise_for_status()`` yields a chainable ``httpx.HTTPStatusError``. + request = httpx.Request(method="POST", url=url) + requester = ScriptedRequester( + [httpx.Response(status_code=429, headers={"Retry-After": "5"}, request=request) for _ in range(max_retries + 1)] + ) + + with ( + caplog.at_level(logging.WARNING, logger=_RETRY_LOG_LOGGER), + pytest.raises(RateLimitError, match="rate-limited") as exc_info, + ): + await _send_request(client_type=client_type, requester=requester, url=url, max_retries=max_retries) + + err = exc_info.value + # Exactly one send more than the retry budget, and it is reflected on the error. + assert requester.call_count == max_retries + 1 + assert err.attempts == max_retries + 1 + assert err.url == url + # ``Retry-After: 5`` was parsed and recorded as the last honoured value. + assert err.retry_after == pytest.approx(5.0) + # The terminal 429 was surfaced as an ``httpx.HTTPStatusError`` and chained as the cause. + assert isinstance(err.__cause__, httpx.HTTPStatusError) + + # One sleep per retry, one WARNING per retry (never on the final, budget-exhausting send). + assert len(recorded_sleeps) == max_retries + + retry_records = [rec for rec in caplog.records if rec.levelno == logging.WARNING and rec.name == _RETRY_LOG_LOGGER] + assert len(retry_records) == max_retries + + logged_attempts: list[int] = [] + for record, expected_delay in zip(retry_records, recorded_sleeps, strict=True): + message = record.getMessage() + assert url in message + match = _RETRY_LOG_PATTERN.search(message) + assert match is not None, message + logged_attempts.append(int(match.group("attempt"))) + # The logged delay is the same value handed to the (patched) sleep for that retry. + assert float(match.group("delay")) == pytest.approx(expected_delay, abs=0.01) + + # Retries are logged in order with a monotonically increasing attempt number. + assert logged_attempts == list(range(1, max_retries + 1)) + + +@pytest.mark.parametrize("client_type", CLIENT_TYPES) +async def test_request_exhausts_raises_rate_limit_error_when_response_has_no_request( + client_type: str, monkeypatch: pytest.MonkeyPatch +) -> None: + """Exhaustion still raises ``RateLimitError`` when the 429 response carries no ``request``. + + A custom ``requester`` may return a response without an attached ``request``; on that + response ``raise_for_status()`` raises ``RuntimeError`` rather than ``httpx.HTTPStatusError``. + The driver must still surface ``RateLimitError`` (never leak the ``RuntimeError``); with no + chainable transport error, ``__cause__`` is ``None``. + """ + _patch_driver_sleep(monkeypatch) + + max_retries = 2 + url = "http://mock/graphql/main" + # No ``request=`` attached, mimicking a hand-built response from a custom requester. + requester = ScriptedRequester([httpx.Response(status_code=429) for _ in range(max_retries + 1)]) + + with pytest.raises(RateLimitError, match="rate-limited") as exc_info: + await _send_request(client_type=client_type, requester=requester, url=url, max_retries=max_retries) + + err = exc_info.value + assert requester.call_count == max_retries + 1 + assert err.attempts == max_retries + 1 + assert err.__cause__ is None + + +@pytest.mark.parametrize("client_type", CLIENT_TYPES) +async def test_request_disabled_surfaces_raw_429_without_retry( + client_type: str, monkeypatch: pytest.MonkeyPatch +) -> None: + """With ``rate_limit_retry_enabled=False`` the driver does ONE send and returns the raw 429. + + The response is returned untouched: no ``RateLimitError`` and no wait. + """ + recorded_sleeps = _patch_driver_sleep(monkeypatch) + + url = "http://mock/graphql/main" + requester = ScriptedRequester([httpx.Response(status_code=429)]) + + if client_type == "standard": + config = Config(address="http://mock", requester=requester.async_request, rate_limit_retry_enabled=False) + client = InfrahubClient(config=config) + response = await client._request(url=url, method=HTTPMethod.POST, headers={}, timeout=10, payload={}) + else: + config = Config(address="http://mock", sync_requester=requester.sync_request, rate_limit_retry_enabled=False) + client_sync = InfrahubClientSync(config=config) + response = client_sync._request(url=url, method=HTTPMethod.POST, headers={}, timeout=10, payload={}) + + # Raw 429 returned untouched: single send, no wait, no RateLimitError. + assert response.status_code == 429 + assert requester.call_count == 1 + assert recorded_sleeps == [] + + +@pytest.mark.parametrize("max_retries", [0, 1, 3]) +@pytest.mark.parametrize("client_type", CLIENT_TYPES) +async def test_request_max_retries_controls_attempt_count( + client_type: str, max_retries: int, monkeypatch: pytest.MonkeyPatch +) -> None: + """A lowered ``rate_limit_max_retries`` bounds the sends: persistent 429 yields ``max_retries + 1``. + + ``max_retries=0`` means no retries — a single 429 send raises ``RateLimitError`` immediately with + zero waits. + """ + recorded_sleeps = _patch_driver_sleep(monkeypatch) + + url = "http://mock/graphql/main" + request = httpx.Request(method="POST", url=url) + requester = ScriptedRequester([httpx.Response(status_code=429, request=request) for _ in range(max_retries + 1)]) + + with pytest.raises(RateLimitError, match="rate-limited") as exc_info: + await _send_request(client_type=client_type, requester=requester, url=url, max_retries=max_retries) + + assert requester.call_count == max_retries + 1 + assert exc_info.value.attempts == max_retries + 1 + # One wait per retry (never on the final, budget-exhausting send). + assert len(recorded_sleeps) == max_retries + + +@dataclass +class ParityCase: + """A single 429 sequence driven identically through the async and sync clients. + + ``build_responses`` returns a fresh scripted response list per client so the two runs are + independent. Every 429 carries ``Retry-After`` so the honoured waits are deterministic (no + jitter), enabling an exact cross-client wait comparison. + """ + + name: str + build_responses: Callable[[httpx.Request], list[httpx.Response]] + max_retries: int + expected_sends: int + expected_waits: list[float] + expect_error: bool + + +_PARITY_SUCCESS_PAYLOAD = {"data": {"result": "success"}} + +PARITY_CASES = [ + ParityCase( + name="retry-after-then-success", + build_responses=lambda request: [ + httpx.Response(status_code=429, headers={"Retry-After": "5"}, request=request), + httpx.Response(status_code=429, headers={"Retry-After": "5"}, request=request), + httpx.Response(status_code=200, json=_PARITY_SUCCESS_PAYLOAD), + ], + max_retries=5, + expected_sends=3, + expected_waits=[5.0, 5.0], + expect_error=False, + ), + ParityCase( + name="retry-after-exhaust", + build_responses=lambda request: [ + httpx.Response(status_code=429, headers={"Retry-After": "5"}, request=request) for _ in range(4) + ], + max_retries=3, + expected_sends=4, + expected_waits=[5.0, 5.0, 5.0], + expect_error=True, + ), +] + + +@pytest.mark.parametrize("case", [pytest.param(tc, id=tc.name) for tc in PARITY_CASES]) +async def test_async_sync_parity_on_identical_429_sequence(case: ParityCase, monkeypatch: pytest.MonkeyPatch) -> None: + """The same 429 sequence yields identical sends, waits, and outcome across both clients. + + Uses a deterministic ``Retry-After``-driven sequence so waits can be compared exactly (rather + than only within jitter tolerance). Asserts identical send counts, matching outcome (same error + type or same success status), and identical honoured waits. + """ + url = "http://mock/graphql/main" + results: dict[str, dict[str, Any]] = {} + + for client_type in CLIENT_TYPES: + request = httpx.Request(method="POST", url=url) + requester = ScriptedRequester(case.build_responses(request)) + recorded_sleeps = _patch_driver_sleep(monkeypatch) + error_type: type | None = None + status: int | None = None + + if case.expect_error: + with pytest.raises(RateLimitError, match="rate-limited") as exc_info: + await _send_request(client_type=client_type, requester=requester, url=url, max_retries=case.max_retries) + error_type = type(exc_info.value) + else: + response = await _send_request( + client_type=client_type, requester=requester, url=url, max_retries=case.max_retries + ) + status = response.status_code + + results[client_type] = { + "sends": requester.call_count, + "waits": list(recorded_sleeps), + "error_type": error_type, + "status": status, + } + + standard = results["standard"] + sync = results["sync"] + + # Identical send counts, matching the expected total. + assert standard["sends"] == sync["sends"] == case.expected_sends + # Same outcome: same error type (or same success status). + assert standard["error_type"] == sync["error_type"] + assert standard["status"] == sync["status"] + # Deterministic Retry-After waits are identical across clients and equal to the expected values. + assert standard["waits"] == pytest.approx(sync["waits"]) + assert standard["waits"] == pytest.approx(case.expected_waits) + + +# --- Backoff growth and jitter divergence at the driver level -------------------------------- +# +# Every retry test above pins the wait with a fixed ``Retry-After``, so ``next_delay`` ignores its +# ``attempt`` argument. A bug that always passed ``attempt=0`` (no exponential growth) would sail +# through the whole suite. The two tests below drive a persistent 429 with NO ``Retry-After`` so the +# wait is driven purely by ``compute_backoff(attempt)``, proving the driver hands an incrementing +# ``attempt`` to ``next_delay`` (growth) and that independent instances jitter differently. + + +async def _send_no_header_429s( + client_type: str, + *, + max_retries: int, + backoff_base: float, + backoff_max: float, +) -> None: + """Drive a persistent, header-less 429 sequence through ``_request`` until the budget is spent. + + Always raises ``RateLimitError`` (the sequence never yields a 200); callers wrap it in + ``pytest.raises``. ``backoff_base``/``backoff_max`` are threaded onto the client ``Config`` so + the recorded waits equal ``compute_backoff(attempt)`` when jitter is neutralised. + """ + url = "http://mock/graphql/main" + request = httpx.Request(method="POST", url=url) + requester = ScriptedRequester([httpx.Response(status_code=429, request=request) for _ in range(max_retries + 1)]) + overrides: dict[str, Any] = { + "rate_limit_max_retries": max_retries, + "rate_limit_backoff_base": backoff_base, + "rate_limit_backoff_max": backoff_max, + } + if client_type == "standard": + config = Config(address="http://mock", requester=requester.async_request, **overrides) + await InfrahubClient(config=config)._request( + url=url, method=HTTPMethod.POST, headers={}, timeout=10, payload={} + ) + return + config = Config(address="http://mock", sync_requester=requester.sync_request, **overrides) + InfrahubClientSync(config=config)._request(url=url, method=HTTPMethod.POST, headers={}, timeout=10, payload={}) + + +@pytest.mark.parametrize("client_type", CLIENT_TYPES) +async def test_backoff_grows_exponentially_and_clamps(client_type: str, monkeypatch: pytest.MonkeyPatch) -> None: + """Header-less persistent 429s wait on exponential backoff that grows per attempt and clamps. + + Jitter is neutralised (``jittered_delay`` patched to the identity) so each recorded wait equals + ``compute_backoff(attempt)``. With ``base=1.0`` and ``max=6.0`` the four retry waits are + ``1, 2, 4, 6`` — doubling until the ceiling clamps the last one. This can only hold if the driver + passes an incrementing ``attempt`` (0, 1, 2, 3) to ``next_delay``. + """ + recorded_sleeps = _patch_driver_sleep(monkeypatch) + # Identity jitter: the recorded wait is exactly the computed backoff ceiling for that attempt. + monkeypatch.setattr(RateLimitRetryHandler, "jittered_delay", lambda _self, ceiling: ceiling) + + max_retries = 4 + backoff_base = 1.0 + backoff_max = 6.0 + + with pytest.raises(RateLimitError, match="rate-limited"): + await _send_no_header_429s( + client_type=client_type, + max_retries=max_retries, + backoff_base=backoff_base, + backoff_max=backoff_max, + ) + + # One wait per retry; base * 2**attempt, doubling then clamped to backoff_max. + assert recorded_sleeps == pytest.approx([1.0, 2.0, 4.0, 6.0]) + + +@pytest.mark.parametrize("client_type", CLIENT_TYPES) +async def test_jitter_differs_between_instances(client_type: str, monkeypatch: pytest.MonkeyPatch) -> None: + """Two independent clients driven through the same header-less 429 sequence jitter differently. + + With real full jitter (``jittered_delay`` NOT patched), the per-retry waits are random draws in + ``[0, compute_backoff(attempt)]``. Across four retries an exact match between two independent + instances is astronomically unlikely, so at least one position must differ. + """ + max_retries = 4 + backoff_base = 5.0 + backoff_max = 60.0 + + runs: list[list[float]] = [] + for _ in range(2): + recorded_sleeps = _patch_driver_sleep(monkeypatch) + with pytest.raises(RateLimitError, match="rate-limited"): + await _send_no_header_429s( + client_type=client_type, + max_retries=max_retries, + backoff_base=backoff_base, + backoff_max=backoff_max, + ) + runs.append(list(recorded_sleeps)) + + first, second = runs + # Both instances performed the same number of jittered waits ... + assert len(first) == len(second) == max_retries + # ... but real full jitter makes at least one recorded wait diverge between the two instances. + assert first != second + + +# --- All-paths coverage and multipart body re-read -------------------------------------------- +# +# ``_request_multipart`` and ``_get_streaming`` build their own ``httpx`` client and BYPASS the +# pluggable ``requester``/``sync_requester`` shim used by the tests above, so their 429->200 +# sequences are scripted at the httpx transport layer with ``httpx_mock`` (pytest-httpx). The +# regular ``_request`` path is exercised the same way here so all three paths share one idiom. + +# A non-empty, multi-line file body large enough that a truncated (unrewound) re-send is obviously +# different from the full payload. +MULTIPART_FILE_CONTENT = b"multipart file body that must survive a 429 retry\n" * 16 + +ALL_REQUEST_PATHS = ["regular", "multipart", "streaming"] + + +def _make_client(client_type: str) -> InfrahubClient | InfrahubClientSync: + """Build a client with no ``requester`` override so real httpx transports (mocked) are used.""" + config = Config(address="http://mock") + if client_type == "standard": + return InfrahubClient(config=config) + return InfrahubClientSync(config=config) + + +def _build_multipart_files() -> dict[str, Any]: + """Build an httpx ``files`` mapping with a non-empty, seekable file object.""" + return {"file": ("upload.bin", io.BytesIO(MULTIPART_FILE_CONTENT), "application/octet-stream")} + + +async def _run_multipart( + client: InfrahubClient | InfrahubClientSync, url: str, files: dict[str, Any] +) -> httpx.Response: + """Drive the real ``_request_multipart`` path on either client.""" + if isinstance(client, InfrahubClient): + return await client._request_multipart(url=url, headers={}, timeout=10, files=files) + return client._request_multipart(url=url, headers={}, timeout=10, files=files) + + +async def _drive_path(client_type: str, path: str, url: str) -> int: + """Drive one request path on the selected client and return the final status code. + + For streaming, the response body is read inside the (async) context manager so the 200 stream + is fully consumed before the status is returned. + """ + client = _make_client(client_type) + + if path == "regular": + if isinstance(client, InfrahubClient): + response = await client._request(url=url, method=HTTPMethod.POST, headers={}, timeout=10, payload={}) + else: + response = client._request(url=url, method=HTTPMethod.POST, headers={}, timeout=10, payload={}) + return response.status_code + + if path == "multipart": + response = await _run_multipart(client=client, url=url, files=_build_multipart_files()) + return response.status_code + + # streaming: retry happens on stream INITIATION (the 429 arrives in the headers before body). + if isinstance(client, InfrahubClient): + async with client._get_streaming(url=url) as response: + assert await response.aread() is not None + return response.status_code + with client._get_streaming(url=url) as response: + assert response.read() is not None + return response.status_code + + +@pytest.mark.parametrize("path", ALL_REQUEST_PATHS) +@pytest.mark.parametrize("client_type", CLIENT_TYPES) +async def test_all_request_paths_retry_429_then_succeed( + client_type: str, path: str, httpx_mock: HTTPXMock, monkeypatch: pytest.MonkeyPatch +) -> None: + """A 429->200 sequence is retried transparently on every request path, both clients. + + Covers the regular request, the multipart upload, and streaming initiation. Each must issue + exactly two transport sends (the retry) and surface the final 200. + """ + recorded_sleeps = _patch_driver_sleep(monkeypatch) + + url = "http://mock/graphql/main" + httpx_mock.add_response(status_code=429) + httpx_mock.add_response(status_code=200, json={"data": {"result": "success"}}) + + status = await _drive_path(client_type=client_type, path=path, url=url) + + # The retry fired: the final 200 is surfaced after exactly two transport sends, with one wait. + assert status == 200 + assert len(httpx_mock.get_requests()) == 2 + assert len(recorded_sleeps) == 1 + + +def _multipart_body_without_boundary(request: httpx.Request) -> bytes: + """Return the multipart body with the random per-request boundary normalised out. + + httpx generates a fresh random boundary for every multipart send, so two identical payloads + still differ byte-for-byte in their boundary markers; normalising it lets us compare the actual + encoded body (headers + file part) across attempts. + """ + content_type = request.headers["content-type"] + _, _, boundary = content_type.partition("boundary=") + return request.content.replace(boundary.encode(), b"__BOUNDARY__") + + +@pytest.mark.parametrize("client_type", CLIENT_TYPES) +async def test_multipart_body_survives_retry( + client_type: str, httpx_mock: HTTPXMock, monkeypatch: pytest.MonkeyPatch +) -> None: + """A retried multipart upload re-sends the FULL file body, not a consumed/empty stream. + + Scripts ``429 -> 200`` for a multipart upload carrying non-empty file content, then captures the + request body the transport received on each attempt. The second attempt must carry the full body + equal to the first (modulo the random multipart boundary), proving the driver rewinds / + re-materialises the payload between attempts. Were the rewind removed, the second send would + stream an already-consumed file object and this test would fail. + """ + _patch_driver_sleep(monkeypatch) + + url = "http://mock/graphql/main" + httpx_mock.add_response(status_code=429) + httpx_mock.add_response(status_code=200, json={"data": {"result": "uploaded"}}) + + client = _make_client(client_type) + response = await _run_multipart(client=client, url=url, files=_build_multipart_files()) + assert response.status_code == 200 + + requests = httpx_mock.get_requests() + assert len(requests) == 2 + + first_body = requests[0].content + second_body = requests[1].content + + # Both attempts carried the full, non-empty file content. + assert MULTIPART_FILE_CONTENT in first_body + assert MULTIPART_FILE_CONTENT in second_body + + # Modulo the random per-request boundary, the retried body is byte-for-byte equal to the first. + assert _multipart_body_without_boundary(requests[0]) == _multipart_body_without_boundary(requests[1]) + + +# --- Direct unit test of the multipart rewind helper ----------------------------------------- +# +# ``test_multipart_body_survives_retry`` above passes even if ``_rewind_multipart_files`` is gutted, +# because httpx itself rewinds seekable files before sending. This exercises the SDK's own helper +# directly so a regression that removes its rewind is caught. + + +class RecordingFile: + """A minimal seekable file object that records every ``seek`` call (no unittest.mock).""" + + def __init__(self, data: bytes) -> None: + self._buffer = io.BytesIO(data) + self.seek_calls: list[int] = [] + + def read(self) -> bytes: + return self._buffer.read() + + def tell(self) -> int: + return self._buffer.tell() + + def seek(self, offset: int, whence: int = io.SEEK_SET) -> int: + self.seek_calls.append(offset) + return self._buffer.seek(offset, whence) + + +def test_rewind_multipart_files_resets_every_file_object() -> None: + """``_rewind_multipart_files`` calls ``seek(0)`` on every file object across the shapes used. + + Covers the ``(filename, fileobj)`` and ``(filename, fileobj, content_type)`` tuple shapes plus a + bare file-object value. Each file is advanced to EOF first; after the rewind every file object + must be back at position 0. This fails if the helper body is gutted. + """ + two_tuple = RecordingFile(b"two-tuple body") + three_tuple = RecordingFile(b"three-tuple body") + bare = RecordingFile(b"bare body") + + files: dict[str, Any] = { + "two": ("two.bin", two_tuple), + "three": ("three.bin", three_tuple, "application/octet-stream"), + "bare": bare, + } + + # Advance every file to EOF so a missing rewind would leave a consumed/empty stream. + for file_obj in (two_tuple, three_tuple, bare): + assert file_obj.read() != b"" + assert file_obj.tell() != 0 + + client_module._rewind_multipart_files(files) + + # Every file object was rewound to the start ... + for file_obj in (two_tuple, three_tuple, bare): + assert file_obj.seek_calls == [0] + assert file_obj.tell() == 0 diff --git a/tests/unit/sdk/test_relogin_headers.py b/tests/unit/sdk/test_relogin_headers.py new file mode 100644 index 000000000..76dd913c3 --- /dev/null +++ b/tests/unit/sdk/test_relogin_headers.py @@ -0,0 +1,94 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + +import pytest + +from infrahub_sdk import Config, InfrahubClient, InfrahubClientSync +from infrahub_sdk.constants import Priority + +if TYPE_CHECKING: + from pytest_httpx import HTTPXMock + +client_types = ["standard", "sync"] + + +def _build_password_client(client_type: str) -> InfrahubClient | InfrahubClientSync: + """Build a password-authenticated client primed with a stale bearer token.""" + config = Config(address="http://mock", username="admin", password="password", insert_tracker=True) + client: InfrahubClient | InfrahubClientSync = ( + InfrahubClient(config=config) if client_type == "standard" else InfrahubClientSync(config=config) + ) + + # Prime the client as if it had already logged in with a now-stale token. + client.access_token = "OLD" + client.refresh_token = "refresh-token" + client.headers["Authorization"] = "Bearer OLD" + return client + + +@pytest.mark.parametrize("client_type", client_types) +async def test_relogin_retry_uses_refreshed_auth_header(client_type: str, httpx_mock: HTTPXMock) -> None: + """The relogin retry carries the freshly-refreshed token, while a per-request priority override rides both attempts. + + The transport helpers merge the per-request delta (here, the priority override) over the + current base headers, so the retry picks up the token refreshed mid-flight instead of a + stale snapshot, and the priority override is preserved across the retry. + """ + # First GraphQL POST returns 401 with an expired-signature error. + httpx_mock.add_response( + method="POST", + url="http://mock/graphql/main", + status_code=401, + json={"errors": [{"message": "Expired Signature"}]}, + ) + # The relogin refresh call issues a NEW access token. + httpx_mock.add_response( + method="POST", + url="http://mock/api/auth/refresh", + json={"access_token": "NEW"}, + ) + # The retried GraphQL POST succeeds; it must carry the refreshed token. + httpx_mock.add_response( + method="POST", + url="http://mock/graphql/main", + json={"data": {"InfrahubInfo": {"version": "1.0"}}}, + ) + + client = _build_password_client(client_type) + query = "query { InfrahubInfo { version }}" + if isinstance(client, InfrahubClient): + await client.execute_graphql(query=query, branch_name="main", priority=Priority.HIGH) + else: + client.execute_graphql(query=query, branch_name="main", priority=Priority.HIGH) + + graphql_requests = [r for r in httpx_mock.get_requests() if str(r.url) == "http://mock/graphql/main"] + assert len(graphql_requests) == 2 + # The first attempt used the stale token; the retry must use the refreshed one. + assert graphql_requests[0].headers["Authorization"] == "Bearer OLD" + assert graphql_requests[1].headers["Authorization"] == "Bearer NEW" + # The per-request priority override rides both the initial attempt and the retry. + assert all(r.headers["x-priority"] == "high" for r in graphql_requests) + + +@pytest.mark.parametrize("client_type", client_types) +def test_merge_request_headers_layers_delta_over_live_base(client_type: str) -> None: + """The merge helper layers the per-request delta over the current base headers. + + A delta with no auth key leaves the live base auth intact (so a refreshed token wins), while + an explicit per-request override — of a normal header or of auth itself — takes precedence. + """ + client = _build_password_client(client_type) + client.headers["X-Priority"] = "medium" + + # A delta carrying only a per-request priority override (no auth). + merged = client._merge_request_headers({"X-Priority": "high"}) + assert merged["X-Priority"] == "high" # per-request override wins over the base default + assert merged["Authorization"] == "Bearer OLD" # base auth is preserved from the live headers + + # After a mid-flight token refresh, a delta with no auth picks up the fresh token. + client.headers["Authorization"] = "Bearer NEW" + assert client._merge_request_headers({"X-Priority": "high"})["Authorization"] == "Bearer NEW" + + # An explicit per-request auth override is respected (the caller owns relogin in that case). + assert client._merge_request_headers({"Authorization": "Bearer CALLER"})["Authorization"] == "Bearer CALLER" diff --git a/tests/unit/sdk/test_repository.py b/tests/unit/sdk/test_repository.py index eca1c9a60..a8364b1c9 100644 --- a/tests/unit/sdk/test_repository.py +++ b/tests/unit/sdk/test_repository.py @@ -13,7 +13,11 @@ @pytest.fixture def temp_dir() -> Generator[str]: - """Fixture to create a temporary directory for testing.""" + """Fixture to create a temporary directory for testing. + + Yields: + str: Path to the temporary directory. + """ with tempfile.TemporaryDirectory() as tmp_dir: yield tmp_dir diff --git a/tests/unit/sdk/test_schema_export.py b/tests/unit/sdk/test_schema_export.py index ed2814e49..e579168c6 100644 --- a/tests/unit/sdk/test_schema_export.py +++ b/tests/unit/sdk/test_schema_export.py @@ -13,6 +13,7 @@ SchemaExport, TemplateSchemaAPI, ) +from infrahub_sdk.schema.export import schema_to_export_dict if TYPE_CHECKING: from pytest_httpx import HTTPXMock @@ -44,8 +45,8 @@ "inherit_from": [], "branch": "aware", "default_filter": None, - "generate_profile": None, - "generate_template": None, + "generate_profile": True, + "generate_template": False, "parent": None, "children": None, "attributes": [], @@ -191,6 +192,25 @@ def test_to_dict(self) -> None: assert len(as_dict["Infra"]["generics"]) == 1 +def test_export_preserves_non_default_ordered_flag() -> None: + """`ordered: false` survives the fetch -> export round-trip; the default `true` is omitted.""" + node = NodeSchemaAPI( + **{ + **_BASE_NODE, + "namespace": "Infra", + "name": "Device", + "attributes": [ + {"name": "tags_unordered", "kind": "List", "ordered": False}, + {"name": "tags_ordered", "kind": "List"}, + ], + } + ) + exported = schema_to_export_dict(node) + attrs = {attr["name"]: attr for attr in exported["attributes"]} + assert attrs["tags_unordered"]["ordered"] is False + assert "ordered" not in attrs["tags_ordered"] + + # --------------------------------------------------------------------------- # Integration tests for export() method on client.schema # --------------------------------------------------------------------------- diff --git a/tests/unit/sdk/test_schema_repository.py b/tests/unit/sdk/test_schema_repository.py index 7c85d7cca..7c1866799 100644 --- a/tests/unit/sdk/test_schema_repository.py +++ b/tests/unit/sdk/test_schema_repository.py @@ -1,11 +1,17 @@ import tempfile +from dataclasses import dataclass, field from pathlib import Path +from typing import Any import pytest +from jsonschema import Draft202012Validator from pydantic import ValidationError from infrahub_sdk.exceptions import FragmentFileNotFoundError, RepositoryFileNotFoundError, ResourceNotDefinedError from infrahub_sdk.schema.repository import ( + MALFORMED_WATCH_MESSAGE, + MISSING_WATCH_MESSAGE, + InfrahubGeneratorDefinitionConfig, InfrahubJinja2TransformConfig, InfrahubPythonTransformConfig, InfrahubRepositoryConfig, @@ -392,3 +398,252 @@ def test_jinja2_transform_payload_includes_declared_watch() -> None: } ) assert config.payload["watch"] == {"files": ["templates/partials/"]} + + +def test_generator_watch_omitted_defaults_to_none() -> None: + """A generator definition without a watch block parses, with watch defaulting to None. + + The generator config sets extra="forbid", so this also proves watch is a recognised optional + field rather than a rejected extra key. + """ + config = InfrahubGeneratorDefinitionConfig.model_validate( + {"name": "my_generator", "file_path": "generators/g.py", "query": "q", "targets": "grp"} + ) + assert config.watch is None + + +def test_generator_watch_parses_object_form() -> None: + """The object form `watch: {files: [...]}` parses into an InfrahubWatchConfig with the files preserved.""" + config = InfrahubGeneratorDefinitionConfig.model_validate( + { + "name": "my_generator", + "file_path": "generators/g.py", + "query": "q", + "targets": "grp", + "watch": {"files": ["a", "dir/"]}, + } + ) + assert config.watch is not None + assert config.watch.files == ["a", "dir/"] + + +def test_generator_watch_list_form_rejected() -> None: + """A bare list `watch: [a, b]` is rejected: the realistic YAML mistake of a list instead of an object. + + Matching the message confirms the rejection is the watch model-type error, not some unrelated failure. + """ + with pytest.raises(ValidationError, match="valid dictionary or instance of InfrahubWatchConfig"): + InfrahubGeneratorDefinitionConfig.model_validate( + { + "name": "my_generator", + "file_path": "generators/g.py", + "query": "q", + "targets": "grp", + "watch": ["a", "b"], + } + ) + + +# --- Advisory 'watch' warning carried by the generated JSON schema --- +# +# The JSON schema generated from InfrahubRepositoryConfig is published to the infrahub-jsonschema +# repository, where YAML language servers use it to validate .infrahub.yml while it is edited. Two +# advisory rules exist purely so editors can nudge: +# +# - Python transforms and generators declare 'watch' as required, flagging an absent block. +# - A 'watch' value must be a mapping, flagging the bare 'watch:' and 'watch: null' forms that +# parse to None and so record nothing, along with any other non-mapping value. +# +# Neither rule is enforced by the models, so these tests pin the split: the JSON schema nudges, the +# runtime stays permissive. Any mapping stays clean, including an empty 'watch: {}' and an explicit +# 'files: []': both record that the author checked and nothing extra needs watching. + +PYTHON_TRANSFORM = {"name": "device_config", "file_path": "transforms/device.py"} +GENERATOR = {"name": "build_interfaces", "file_path": "generators/iface.py", "query": "q", "targets": "grp"} +JINJA2_TRANSFORM = {"name": "device_config", "query": "q", "template_path": "templates/device.j2"} + + +def _validation_errors(document: dict[str, Any]) -> list[Any]: + return list(Draft202012Validator(InfrahubRepositoryConfig.model_json_schema()).iter_errors(document)) + + +def missing_watch_paths(document: dict[str, Any]) -> list[str]: + """Definitions in ``document`` the schema flags for having no 'watch' block at all. + + Filtering on ``validator_value`` isolates the advisory requirement from the genuine required + fields, so an unrelated omission elsewhere in the document cannot be mistaken for a watch warning. + """ + return [ + "/".join(str(part) for part in error.absolute_path) + for error in _validation_errors(document) + if error.validator == "required" and error.validator_value == ["watch"] + ] + + +def flagged_watch_paths(document: dict[str, Any]) -> list[str]: + """'watch' values in ``document`` the schema flags for not being a mapping. + + Anchored on the location rather than the keyword, so it catches every non-mapping value however + it fails. Duplicates are collapsed because one such value can fail several keywords at once. + """ + paths = [ + "/".join(str(part) for part in error.absolute_path) + for error in _validation_errors(document) + if error.absolute_path and error.absolute_path[-1] == "watch" + ] + return list(dict.fromkeys(paths)) + + +@dataclass +class WatchWarningCase: + name: str + document: dict[str, Any] + missing: list[str] = field(default_factory=list) + flagged: list[str] = field(default_factory=list) + + +WATCH_WARNING_CASES = [ + WatchWarningCase( + name="python-transform-without-watch", + document={"python_transforms": [PYTHON_TRANSFORM]}, + missing=["python_transforms/0"], + ), + WatchWarningCase( + name="generator-without-watch", + document={"generator_definitions": [GENERATOR]}, + missing=["generator_definitions/0"], + ), + WatchWarningCase( + name="each-entry-flagged-independently", + document={ + "python_transforms": [PYTHON_TRANSFORM | {"watch": {"files": ["lib/"]}}, PYTHON_TRANSFORM], + "generator_definitions": [GENERATOR], + }, + missing=["python_transforms/1", "generator_definitions/0"], + ), + WatchWarningCase( + name="bare-watch-key-is-not-an-answer", + document={"python_transforms": [PYTHON_TRANSFORM | {"watch": None}]}, + flagged=["python_transforms/0/watch"], + ), + WatchWarningCase( + name="empty-watch-block-is-a-deliberate-choice", + document={"python_transforms": [PYTHON_TRANSFORM | {"watch": {}}]}, + ), + WatchWarningCase( + name="empty-generator-watch-block-is-a-deliberate-choice", + document={"generator_definitions": [GENERATOR | {"watch": {}}]}, + ), + WatchWarningCase( + name="watch-as-a-bare-list", + document={"python_transforms": [PYTHON_TRANSFORM | {"watch": ["lib/"]}]}, + flagged=["python_transforms/0/watch"], + ), + WatchWarningCase( + name="watch-as-a-string", + document={"python_transforms": [PYTHON_TRANSFORM | {"watch": "lib/"}]}, + flagged=["python_transforms/0/watch"], + ), + WatchWarningCase( + name="python-transform-with-watch", + document={"python_transforms": [PYTHON_TRANSFORM | {"watch": {"files": ["lib/helpers.py"]}}]}, + ), + WatchWarningCase( + name="generator-with-watch", + document={"generator_definitions": [GENERATOR | {"watch": {"files": ["lib/"]}}]}, + ), + WatchWarningCase( + name="empty-files-list-is-a-deliberate-choice", + document={"python_transforms": [PYTHON_TRANSFORM | {"watch": {"files": []}}]}, + ), + WatchWarningCase( + name="jinja2-transform-without-watch-is-out-of-scope", + document={"jinja2_transforms": [JINJA2_TRANSFORM]}, + ), + WatchWarningCase( + name="check-definition-has-no-watch-to-warn-about", + document={"check_definitions": [{"name": "my_check", "file_path": "check.py"}]}, + ), +] + + +@pytest.mark.parametrize("case", [pytest.param(tc, id=tc.name) for tc in WATCH_WARNING_CASES]) +def test_watch_warnings_flagged_by_json_schema(case: WatchWarningCase) -> None: + assert missing_watch_paths(case.document) == case.missing + assert flagged_watch_paths(case.document) == case.flagged + + +def test_watch_warnings_carry_their_guidance_messages() -> None: + """Both rules ship 'errorMessage', which is what a YAML language server displays. + + Without them editors fall back to a bare `Missing property "watch"` or `Incorrect type`, neither + of which tells anyone why they should care. + """ + defs = InfrahubRepositoryConfig.model_json_schema()["$defs"] + for name in ("InfrahubPythonTransformConfig", "InfrahubGeneratorDefinitionConfig"): + assert defs[name]["allOf"] == [{"required": ["watch"], "errorMessage": MISSING_WATCH_MESSAGE}] + for name in ("InfrahubPythonTransformConfig", "InfrahubGeneratorDefinitionConfig", "InfrahubJinja2TransformConfig"): + watch = defs[name]["properties"]["watch"] + assert watch["type"] == "object" + assert watch["errorMessage"] == MALFORMED_WATCH_MESSAGE + + +def test_empty_watch_block_records_the_acknowledgement_a_bare_key_does_not() -> None: + """The runtime difference is the whole reason only one of the two forms warns. + + 'files' defaults to an empty list, so 'watch: {}' still produces a watch config and carries the + author's "nothing extra needs watching" either way. A bare 'watch:' parses to None, which is + indistinguishable from never having declared it, so nothing is recorded and the schema flags it. + Collapsing that difference would make one of the two warnings wrong. + """ + empty_block = InfrahubPythonTransformConfig.model_validate(PYTHON_TRANSFORM | {"watch": {}}) + bare_key = InfrahubPythonTransformConfig.model_validate(PYTHON_TRANSFORM | {"watch": None}) + + assert empty_block.watch is not None + assert empty_block.watch.files == [] + assert bare_key.watch is None + + +def test_genuine_required_fields_survive_alongside_the_watch_warning() -> None: + """The advisory requirement must not displace the fields pydantic marks as required. + + Declaring it through a top-level 'required' in json_schema_extra would overwrite the generated + list, silently making genuinely mandatory fields optional in the published schema. + """ + defs = InfrahubRepositoryConfig.model_json_schema()["$defs"] + assert defs["InfrahubPythonTransformConfig"]["required"] == ["name", "file_path"] + assert defs["InfrahubGeneratorDefinitionConfig"]["required"] == ["name", "file_path", "query", "targets"] + + +def test_watch_field_keeps_its_nullable_reference() -> None: + """Narrowing the field to an object must not replace the reference pydantic generates. + + The 'type' keyword sits alongside the anyOf so a well-formed block still validates against + InfrahubWatchConfig, which is what drives editor completion inside the block. + """ + watch = InfrahubRepositoryConfig.model_json_schema()["$defs"]["InfrahubPythonTransformConfig"]["properties"][ + "watch" + ] + assert watch["anyOf"] == [{"$ref": "#/$defs/InfrahubWatchConfig"}, {"type": "null"}] + + +def test_repository_json_schema_is_a_valid_draft_2020_12_schema() -> None: + Draft202012Validator.check_schema(InfrahubRepositoryConfig.model_json_schema()) + + +@pytest.mark.parametrize( + "watch", + [pytest.param(None, id="watch-omitted"), pytest.param({"watch": None}, id="watch-explicitly-null")], +) +def test_warned_about_watch_forms_stay_valid_at_runtime(watch: dict[str, Any] | None) -> None: + """The warnings are editor-only. + + The schema is deliberately stricter than the models here, so parsing a config the editor warns + about must keep working. + """ + extra = watch or {} + config = InfrahubRepositoryConfig.model_validate( + {"python_transforms": [PYTHON_TRANSFORM | extra], "generator_definitions": [GENERATOR | extra]} + ) + assert config.python_transforms[0].watch is None + assert config.generator_definitions[0].watch is None diff --git a/tests/unit/sdk/test_task.py b/tests/unit/sdk/test_task.py index dd029003f..c3e26c698 100644 --- a/tests/unit/sdk/test_task.py +++ b/tests/unit/sdk/test_task.py @@ -1,13 +1,15 @@ from __future__ import annotations +import json from datetime import datetime, timezone from typing import TYPE_CHECKING import pytest +from infrahub_sdk.graphql import Mutation from infrahub_sdk.task.exceptions import TaskNotFoundError, TooManyTasksError -from infrahub_sdk.task.manager import InfraHubTaskManagerBase -from infrahub_sdk.task.models import Task, TaskFilter, TaskState +from infrahub_sdk.task.manager import MUTATION_TASK_QUERY, InfraHubTaskManagerBase +from infrahub_sdk.task.models import Task, TaskFilter, TaskState, WebhookDeliveryTask if TYPE_CHECKING: from pytest_httpx import HTTPXMock @@ -39,6 +41,95 @@ async def test_method_all_full(clients: BothClients, mock_query_tasks_01: HTTPXM assert isinstance(tasks[0], Task) +@pytest.mark.parametrize("client_type", client_types) +async def test_method_retry(clients: BothClients, httpx_mock: HTTPXMock, client_type: str) -> None: + httpx_mock.add_response( + method="POST", + json={"data": {"InfrahubTaskRetry": {"ok": True, "task": {"id": "b71f5542-7b54-562f-9053-8fd6ec0b0481"}}}}, + match_headers={"X-Infrahub-Tracker": "mutation-task-retry"}, + ) + + if client_type == "standard": + new_id = await clients.standard.task.retry(id="a60f4431-6a43-451e-8f42-9ec5db9a9370") + else: + new_id = clients.sync.task.retry(id="a60f4431-6a43-451e-8f42-9ec5db9a9370") + + assert new_id == "b71f5542-7b54-562f-9053-8fd6ec0b0481" + sent_query = json.loads(httpx_mock.get_requests()[-1].content)["query"] + assert "InfrahubTaskRetry(" in sent_query + assert 'id: "a60f4431-6a43-451e-8f42-9ec5db9a9370"' in sent_query + + +@pytest.mark.parametrize("client_type", client_types) +async def test_method_cancel(clients: BothClients, httpx_mock: HTTPXMock, client_type: str) -> None: + httpx_mock.add_response( + method="POST", + json={"data": {"InfrahubTaskCancel": {"ok": True, "task": {"id": "a60f4431-6a43-451e-8f42-9ec5db9a9370"}}}}, + match_headers={"X-Infrahub-Tracker": "mutation-task-cancel"}, + ) + + if client_type == "standard": + cancelled = await clients.standard.task.cancel(id="a60f4431-6a43-451e-8f42-9ec5db9a9370") + else: + cancelled = clients.sync.task.cancel(id="a60f4431-6a43-451e-8f42-9ec5db9a9370") + + assert cancelled is True + sent_query = json.loads(httpx_mock.get_requests()[-1].content)["query"] + assert "InfrahubTaskCancel(" in sent_query + assert 'id: "a60f4431-6a43-451e-8f42-9ec5db9a9370"' in sent_query + + +@pytest.mark.parametrize("client_type", client_types) +async def test_filter_limit_forwards_include_actions( + clients: BothClients, mock_query_tasks_03: HTTPXMock, client_type: str +) -> None: + if client_type == "standard": + await clients.standard.task.filter(limit=5, include_actions=True) + else: + clients.sync.task.filter(limit=5, include_actions=True) + + sent_query = json.loads(mock_query_tasks_03.get_requests()[-1].content)["query"] + assert "available_actions" in sent_query + + +@pytest.mark.parametrize("client_type", client_types) +async def test_filter_limit_forwards_include_diagnostics( + clients: BothClients, mock_query_tasks_03: HTTPXMock, client_type: str +) -> None: + if client_type == "standard": + await clients.standard.task.filter(limit=5, include_diagnostics=True) + else: + clients.sync.task.filter(limit=5, include_diagnostics=True) + + sent_query = json.loads(mock_query_tasks_03.get_requests()[-1].content)["query"] + assert "... on WebhookDeliveryTask" in sent_query + + +async def test_action_mutation_render() -> None: + query = Mutation( + mutation="InfrahubTaskRetry", + input_data={"data": {"id": "a60f4431-6a43-451e-8f42-9ec5db9a9370"}}, + query=MUTATION_TASK_QUERY, + ) + assert ( + query.render() + == """ +mutation { + InfrahubTaskRetry( + data: { + id: "a60f4431-6a43-451e-8f42-9ec5db9a9370" + } + ){ + ok + task { + id + } + } +} +""" + ) + + async def test_generate_count_query() -> None: query = InfraHubTaskManagerBase._generate_count_query() assert query @@ -121,8 +212,10 @@ async def test_method_get_full(clients: BothClients, mock_query_tasks_05: HTTPXM assert len(task.logs) == 4 assert len(task.related_nodes) == 2 assert task.model_dump() == { + "available_actions": [], "branch": "main", "created_at": datetime(2025, 1, 18, 22, 12, 20, 228112, tzinfo=timezone.utc), + "error": None, "id": "32116fcd-9071-43a7-9f14-777901020b5b", "logs": [ { @@ -161,3 +254,90 @@ async def test_method_get_full(clients: BothClients, mock_query_tasks_05: HTTPXM "updated_at": datetime(2025, 1, 18, 22, 12, 22, 44921, tzinfo=timezone.utc), "workflow": "import-python-files", } + + +def _base_task_data() -> dict: + return { + "id": "a60f4431-6a43-451e-8f42-9ec5db9a9370", + "title": "Webhook delivery", + "state": "COMPLETED", + "created_at": "2025-01-18T22:12:20.228112+00:00", + "updated_at": "2025-01-18T22:12:22.044921+00:00", + } + + +async def test_available_actions_parsed() -> None: + task = Task.from_graphql( + { + **_base_task_data(), + "available_actions": [ + {"action": "RETRY", "available": True, "unavailability_reason": None}, + {"action": "CANCEL", "available": False, "unavailability_reason": "the task has already settled"}, + ], + } + ) + + assert task.can_retry is True + assert task.can_cancel is False + assert task.available_actions[1].unavailability_reason == "the task has already settled" + + +async def test_available_actions_absent_defaults_empty() -> None: + task = Task.from_graphql(_base_task_data()) + + assert task.available_actions == [] + assert task.can_retry is False + assert task.can_cancel is False + + +async def test_error_parsed() -> None: + task = Task.from_graphql( + { + **_base_task_data(), + "error": { + "status_class": "client_error", + "message": "endpoint returned 404", + "remediation": "check the webhook URL", + }, + } + ) + + assert task.error is not None + assert task.error.status_class == "client_error" + assert task.error.message == "endpoint returned 404" + assert task.error.remediation == "check the webhook URL" + + +async def test_webhook_send_dispatches_to_webhook_task() -> None: + task = Task.from_graphql( + { + **_base_task_data(), + "workflow": "webhook-send", + "http_request": {"url": "https://example.com/hook", "headers": {"X-Signature": "***"}}, + "http_response": {"status_code": 200, "body": "ok", "latency_ms": 42.5}, + } + ) + + assert isinstance(task, WebhookDeliveryTask) + assert task.http_request is not None + assert task.http_request.url == "https://example.com/hook" + assert task.http_response is not None + assert task.http_response.status_code == 200 + assert task.http_response.latency_ms == pytest.approx(42.5) + + +async def test_generate_query_excludes_diagnostics_by_default() -> None: + query = InfraHubTaskManagerBase._generate_query().render() + + assert "error {" not in query + assert "... on WebhookDeliveryTask {" not in query + + +async def test_generate_query_includes_diagnostics_when_requested() -> None: + query = InfraHubTaskManagerBase._generate_query(include_diagnostics=True).render() + + assert "error {" in query + assert "remediation" in query + assert "... on WebhookDeliveryTask {" in query + assert "http_request {" in query + assert "http_response {" in query diff --git a/tests/unit/test_rate_limit.py b/tests/unit/test_rate_limit.py new file mode 100644 index 000000000..08e32afb6 --- /dev/null +++ b/tests/unit/test_rate_limit.py @@ -0,0 +1,133 @@ +"""Unit tests for the pure ``RateLimitRetryHandler`` decision logic. + +These tests exercise the handler in isolation (no I/O, no sleeping): exponential-backoff +growth and clamping, full-jitter bounds, ``Retry-After`` parsing (delta-seconds, HTTP-date, +past dates, malformed input), ``next_delay`` selection/clamping, and the retry budget. +""" + +from __future__ import annotations + +from datetime import datetime, timedelta, timezone + +import pytest + +from infrahub_sdk.rate_limit import RateLimitRetryHandler + + +def make_handler(max_retries: int = 5, backoff_base: float = 0.5, backoff_max: float = 60.0) -> RateLimitRetryHandler: + return RateLimitRetryHandler(max_retries=max_retries, backoff_base=backoff_base, backoff_max=backoff_max) + + +def test_compute_backoff_grows_exponentially() -> None: + handler = make_handler(backoff_base=0.5, backoff_max=60.0) + assert handler.compute_backoff(0) == pytest.approx(0.5) + assert handler.compute_backoff(1) == pytest.approx(1.0) + assert handler.compute_backoff(2) == pytest.approx(2.0) + assert handler.compute_backoff(3) == pytest.approx(4.0) + # Monotonic non-decreasing. + values = [handler.compute_backoff(attempt) for attempt in range(8)] + assert values == sorted(values) + + +def test_compute_backoff_clamped_to_backoff_max() -> None: + handler = make_handler(backoff_base=0.5, backoff_max=10.0) + # 0.5 * 2**10 = 512 -> clamped to 10.0 + assert handler.compute_backoff(10) == pytest.approx(10.0) + assert handler.compute_backoff(20) == pytest.approx(10.0) + + +def test_jittered_delay_within_bounds() -> None: + handler = make_handler() + for _ in range(100): + delay = handler.jittered_delay(4.0) + assert 0.0 <= delay <= 4.0 + + +def test_jittered_delay_varies() -> None: + handler = make_handler() + draws = {handler.jittered_delay(10.0) for _ in range(50)} + # A sample of full-jitter draws should not all be identical. + assert len(draws) > 1 + + +def test_parse_retry_after_delta_seconds() -> None: + handler = make_handler() + assert handler.parse_retry_after("30") == pytest.approx(30.0) + assert handler.parse_retry_after("0") == pytest.approx(0.0) + + +def test_parse_retry_after_http_date() -> None: + handler = make_handler() + now = datetime(2026, 7, 7, 12, 0, 0, tzinfo=timezone.utc) + future = now + timedelta(seconds=120) + header = future.strftime("%a, %d %b %Y %H:%M:%S GMT") + assert handler.parse_retry_after(header, now=now) == pytest.approx(120.0, abs=1.0) + + +def test_parse_retry_after_past_date_is_zero() -> None: + handler = make_handler() + now = datetime(2026, 7, 7, 12, 0, 0, tzinfo=timezone.utc) + past = now - timedelta(seconds=120) + header = past.strftime("%a, %d %b %Y %H:%M:%S GMT") + assert handler.parse_retry_after(header, now=now) == pytest.approx(0.0) + + +@pytest.mark.parametrize("header", [None, "", " ", "not-a-date", "12.5.6"]) +def test_parse_retry_after_malformed_returns_none(header: str | None) -> None: + handler = make_handler() + assert handler.parse_retry_after(header) is None + + +def test_parse_retry_after_negative_delta_floored_to_zero() -> None: + handler = make_handler() + # A negative delta-seconds value must floor at 0.0, never a negative wait (which would make + # the sync driver's time.sleep raise ValueError while asyncio.sleep would tolerate it). + assert handler.parse_retry_after("-5") == pytest.approx(0.0) + # ... and the floor propagates through next_delay, so no negative delay ever reaches sleep. + assert handler.next_delay(attempt=0, retry_after_header="-5") == pytest.approx(0.0) + + +def test_parse_retry_after_pathological_huge_value_returns_none() -> None: + handler = make_handler() + # An arbitrarily long digit string overflows float(int(value)); this must fall back to + # computed backoff (None) rather than raising OverflowError and crashing the request. + assert handler.parse_retry_after("9" * 5000) is None + + +def test_next_delay_honours_retry_after_clamped() -> None: + handler = make_handler(backoff_max=60.0) + assert handler.next_delay(attempt=0, retry_after_header="10") == pytest.approx(10.0) + # Retry-After larger than backoff_max is clamped. + assert handler.next_delay(attempt=0, retry_after_header="600") == pytest.approx(60.0) + + +def test_next_delay_falls_back_to_jittered_backoff() -> None: + handler = make_handler(backoff_base=2.0, backoff_max=60.0) + for _ in range(50): + delay = handler.next_delay(attempt=3) + # compute_backoff(3) = 16.0 -> jittered in [0, 16]. + assert 0.0 <= delay <= 16.0 + + +def test_next_delay_result_always_clamped() -> None: + handler = make_handler(backoff_base=1000.0, backoff_max=5.0) + for _ in range(50): + assert handler.next_delay(attempt=5) <= 5.0 + + +def test_should_retry_yields_max_retries_plus_one_total_sends() -> None: + max_retries = 5 + handler = make_handler(max_retries=max_retries) + attempts = 0 + # Simulate a driver loop that always receives 429. + while True: + attempts += 1 # one send performed + if not handler.should_retry(attempts_made=attempts): + break + assert attempts == max_retries + 1 + + +def test_should_retry_zero_retries() -> None: + handler = make_handler(max_retries=0) + # After the single initial send, no retry is allowed. + assert handler.should_retry(attempts_made=1) is False diff --git a/tests/unit/test_schema_generated_models.py b/tests/unit/test_schema_generated_models.py new file mode 100644 index 000000000..7e7eca521 --- /dev/null +++ b/tests/unit/test_schema_generated_models.py @@ -0,0 +1,240 @@ +"""Drift guard for the generated user-facing write/read schema models. + +The SDK repo cannot regenerate these models on its own (they are rendered from the +backend's schema definitions), so this checks what the SDK can verify standalone: +the generated files are present, carry the do-not-edit header, expose the expected +model families, and satisfy the write/read structural invariants (write drops extra +fields silently; read is a superset of write). The full regeneration drift is enforced by the +monorepo's generated-file CI, which regenerates and fails on any diff. + +The attribute family is a discriminated union on ``kind``: a shared ``AttributeSchemaBase`` +carries every field except ``parameters``, and each variant narrows ``kind`` and adds its own +``parameters`` model. The public ``AttributeSchema{Write,Read}`` name is the union alias, so +class-level checks introspect the base and the variants rather than the alias. +""" + +from __future__ import annotations + +from enum import Enum +from pathlib import Path +from typing import TYPE_CHECKING + +import pytest + +from infrahub_sdk.schema import InfrahubSchemaRead, InfrahubSchemaWrite +from infrahub_sdk.schema.generated import enums as enums_module +from infrahub_sdk.schema.generated import read as read_module +from infrahub_sdk.schema.generated import write as write_module + +if TYPE_CHECKING: + from pydantic import BaseModel + +_GENERATED_DIR = Path(write_module.__file__).parent +# Plain (non-union) families pair their write-variant class name with their read-variant class name. +_FAMILY_PAIRS = [ + ("RelationshipSchemaWrite", "RelationshipSchemaRead"), + ("BaseNodeSchemaWrite", "BaseNodeSchemaRead"), + ("NodeSchemaWrite", "NodeSchemaRead"), + ("GenericSchemaWrite", "GenericSchemaRead"), +] +_WRITE_FAMILIES = [write for write, _ in _FAMILY_PAIRS] + +# The attribute union's shared base plus its kind-discriminated variants (bare names; a variant's +# class is ``Write`` / ``Read``). +_ATTRIBUTE_BASE = "AttributeSchemaBase" +_ATTRIBUTE_VARIANTS = [ + "TextAttribute", + "NumberAttribute", + "ListAttribute", + "NumberPoolAttribute", + "GenericAttribute", +] + + +def _attribute_write_classes() -> list[type[BaseModel]]: + return [getattr(write_module, _ATTRIBUTE_BASE + "Write")] + [ + getattr(write_module, variant + "Write") for variant in _ATTRIBUTE_VARIANTS + ] + + +def _attribute_read_classes() -> list[type[BaseModel]]: + return [getattr(read_module, _ATTRIBUTE_BASE + "Read")] + [ + getattr(read_module, variant + "Read") for variant in _ATTRIBUTE_VARIANTS + ] + + +def _aggregate_fields(models: list[type[BaseModel]]) -> set[str]: + fields: set[str] = set() + for model in models: + fields |= set(model.model_fields) + return fields + + +@pytest.mark.parametrize("filename", ["write.py", "read.py"]) +def test_generated_files_present_with_do_not_edit_header(filename: str) -> None: + path = _GENERATED_DIR / filename + assert path.is_file(), f"Generated schema model file is missing: {path}" + assert "do not edit" in path.read_text().splitlines()[0].lower() + + +@pytest.mark.parametrize(("write_family", "read_family"), _FAMILY_PAIRS) +def test_expected_model_families_present_in_each_variant(write_family: str, read_family: str) -> None: + assert hasattr(write_module, write_family), f"write variant is missing {write_family}" + assert hasattr(read_module, read_family), f"read variant is missing {read_family}" + + +def test_attribute_union_families_present_in_each_variant() -> None: + # The public union alias and every variant (plus the shared base) must exist in both variants. + for name in ["AttributeSchema", _ATTRIBUTE_BASE, *_ATTRIBUTE_VARIANTS]: + assert hasattr(write_module, name + "Write"), f"write variant is missing {name}Write" + assert hasattr(read_module, name + "Read"), f"read variant is missing {name}Read" + + +@pytest.mark.parametrize("family", _WRITE_FAMILIES) +def test_write_variant_ignores_extra_fields(family: str) -> None: + model: type[BaseModel] = getattr(write_module, family) + assert model.model_config.get("extra") == "ignore", ( + f"write variant {family} must set extra='ignore' so non-settable fields are dropped, not rejected" + ) + + +def test_attribute_write_base_and_variants_ignore_extra_fields() -> None: + for model in _attribute_write_classes(): + assert model.model_config.get("extra") == "ignore", ( + f"write attribute model {model.__name__} must set extra='ignore'" + ) + + +@pytest.mark.parametrize(("write_family", "read_family"), _FAMILY_PAIRS) +def test_read_variant_is_superset_of_write_variant(write_family: str, read_family: str) -> None: + write_model: type[BaseModel] = getattr(write_module, write_family) + read_model: type[BaseModel] = getattr(read_module, read_family) + write_fields = set(write_model.model_fields) + read_fields = set(read_model.model_fields) + missing = write_fields - read_fields + assert not missing, f"read variant {read_family} must expose every write field; missing: {sorted(missing)}" + + +def test_attribute_read_is_superset_of_attribute_write() -> None: + # Aggregated across the base and every variant, read must expose every write field. + write_fields = _aggregate_fields(_attribute_write_classes()) + read_fields = _aggregate_fields(_attribute_read_classes()) + missing = write_fields - read_fields + assert not missing, f"read attribute variants must expose every write field; missing: {sorted(missing)}" + + +def test_write_variant_carries_read_level_fields_absent() -> None: + # `inherited` is a read-level attribute field carried on the shared base; it must be absent on + # the write base and present on the read base. + write_base = getattr(write_module, _ATTRIBUTE_BASE + "Write") + read_base = getattr(read_module, _ATTRIBUTE_BASE + "Read") + assert "inherited" not in write_base.model_fields + assert "inherited" in read_base.model_fields + + +def test_attribute_parameters_only_on_variants_not_base() -> None: + # `parameters` is what the union discriminates, so it lives on the variants, not the base. + write_base = getattr(write_module, _ATTRIBUTE_BASE + "Write") + assert "parameters" not in write_base.model_fields + for variant in _ATTRIBUTE_VARIANTS: + model = getattr(write_module, variant + "Write") + assert "parameters" in model.model_fields, f"{model.__name__} must carry a parameters field" + + +def test_document_root_models_import_with_nodes_and_generics() -> None: + for root in (InfrahubSchemaWrite, InfrahubSchemaRead): + assert "nodes" in root.model_fields, f"{root.__name__} must expose a 'nodes' field" + assert "generics" in root.model_fields, f"{root.__name__} must expose a 'generics' field" + # The write root drops unknown top-level keys silently (tolerated, not rejected). + assert InfrahubSchemaWrite.model_config.get("extra") == "ignore" + + +def test_write_root_exposes_extensions_field() -> None: + # Extensions are part of the write contract (write-only); the read root does not carry them. + assert "extensions" in InfrahubSchemaWrite.model_fields + assert "extensions" not in InfrahubSchemaRead.model_fields + + +@pytest.mark.parametrize("name", ["NodeExtensionWrite", "SchemaExtensionWrite"]) +def test_extension_models_present_on_write_variant_only(name: str) -> None: + assert hasattr(write_module, name), f"write variant is missing {name}" + assert not hasattr(read_module, name.replace("Write", "Read")), ( + f"extension models are write-only; read variant must not define {name.replace('Write', 'Read')}" + ) + + +@pytest.mark.parametrize("name", ["NodeExtensionWrite", "SchemaExtensionWrite"]) +def test_extension_models_ignore_extra_fields(name: str) -> None: + model: type[BaseModel] = getattr(write_module, name) + assert model.model_config.get("extra") == "ignore", f"extension model {name} must set extra='ignore'" + + +# Constrained fields are typed with dedicated (str, Enum) classes emitted into enums.py rather +# than inline Literals. Each generated enum's ordered values must match this contract. +_EXPECTED_ENUM_VALUES = { + "BranchSupportType": ["aware", "agnostic", "local"], + "RelationshipKind": ["Generic", "Attribute", "Component", "Parent", "Group", "Hierarchy", "Profile", "Template"], + "RelationshipCardinality": ["one", "many"], + "RelationshipDirection": ["bidirectional", "outbound", "inbound"], + "RelationshipDeleteBehavior": ["no-action", "cascade"], + "AllowOverrideType": ["none", "any"], + "SchemaState": ["present", "absent"], + "SchemaAttributeDisplay": ["default", "extra"], + "ComputedAttributeKind": ["User", "Jinja2", "TransformPython"], +} + + +@pytest.mark.parametrize("enum_name", sorted(_EXPECTED_ENUM_VALUES)) +def test_generated_enums_are_str_enum_classes_with_expected_values(enum_name: str) -> None: + enum_cls = getattr(enums_module, enum_name) + assert issubclass(enum_cls, Enum), f"{enum_name} must be an Enum class" + assert issubclass(enum_cls, str), f"{enum_name} must be a str-backed enum" + assert [member.value for member in enum_cls] == _EXPECTED_ENUM_VALUES[enum_name] + + +def test_attribute_kind_enum_present_without_deprecated_string_member() -> None: + assert issubclass(enums_module.AttributeKind, Enum) + assert issubclass(enums_module.AttributeKind, str) + values = [member.value for member in enums_module.AttributeKind] + # The deprecated "String" kind is dropped from the generated enum. + assert "String" not in values + assert "Text" in values + + +def test_constrained_fields_are_typed_with_generated_enums() -> None: + # The constrained fields reference the dedicated enum classes, not inline Literals. + assert ( + write_module.RelationshipSchemaWrite.model_fields["cardinality"].annotation + is enums_module.RelationshipCardinality + ) + assert write_module.RelationshipSchemaWrite.model_fields["kind"].annotation is enums_module.RelationshipKind + assert write_module.AttributeSchemaBaseWrite.model_fields["kind"].annotation is enums_module.AttributeKind + assert ( + read_module.RelationshipSchemaRead.model_fields["cardinality"].annotation + is enums_module.RelationshipCardinality + ) + + +def test_use_enum_values_keeps_runtime_field_values_as_plain_strings() -> None: + # use_enum_values means a constructed model stores the plain string, so equality against + # both the raw string and the enum member holds and serialization is unchanged. + # Passing the raw string is intentional here: the field is typed as the enum, but the point of + # this test is that pydantic coerces a plain string at runtime, so the static complaint is expected. + relationship = write_module.RelationshipSchemaWrite(name="interfaces", peer="InfraInterface", cardinality="one") # ty: ignore[invalid-argument-type] + assert relationship.cardinality == "one" + assert relationship.cardinality == enums_module.RelationshipCardinality.ONE + assert isinstance(relationship.cardinality, str) + # Discriminates the mode: without use_enum_values the value would be a RelationshipCardinality + # member (also a str, so the assertions above cannot tell the modes apart). A plain string is + # not an instance of the enum, so this fails if use_enum_values is ever dropped on regeneration. + assert not isinstance(relationship.cardinality, enums_module.RelationshipCardinality) + + +@pytest.mark.parametrize("name", ["ProfileSchemaRead", "TemplateSchemaRead"]) +def test_profile_template_read_models_present_on_read_variant_only(name: str) -> None: + # Profiles and templates are read-only projections; only the read variant defines them. + model: type[BaseModel] = getattr(read_module, name) + assert "inherit_from" in model.model_fields, f"{name} must expose an 'inherit_from' field" + assert not hasattr(write_module, name.replace("Read", "Write")), ( + f"profile/template models are read-only; write variant must not define {name.replace('Read', 'Write')}" + ) diff --git a/tests/unit/test_schema_offline_validation.py b/tests/unit/test_schema_offline_validation.py new file mode 100644 index 000000000..d9353672c --- /dev/null +++ b/tests/unit/test_schema_offline_validation.py @@ -0,0 +1,556 @@ +"""Offline schema validation: with only the SDK installed (pydantic, no server). + +Validates a schema payload against the generated write models and asserts the +field-level verdict, without importing the backend/server package. Values the user may not +set never reach the server, but they are reported: a read-only field -- one the read API +returns -- as a warning, and any other extra field as an error. Enum, constraint and +required-field violations are reported naming the field and the invalid value. +""" + +from __future__ import annotations + +from dataclasses import dataclass + +import pytest + +from infrahub_sdk.schema import InfrahubSchemaRead, InfrahubSchemaWrite, validate_schema +from infrahub_sdk.schema.validate import SchemaValidationResult + + +def _valid_schema() -> dict: + return { + "version": "1.0", + "nodes": [ + { + "name": "Device", + "namespace": "Infra", + "attributes": [ + {"name": "hostname", "kind": "Text"}, + {"name": "count", "kind": "Number", "optional": True}, + ], + "relationships": [ + {"name": "interfaces", "peer": "InfraInterface", "cardinality": "many", "optional": True}, + ], + }, + ], + "generics": [ + {"name": "Endpoint", "namespace": "Infra", "attributes": [{"name": "role", "kind": "Text"}]}, + ], + } + + +def _fields_named(result: SchemaValidationResult) -> set[str]: + return {error.field for error in result.errors} + + +def _extension_node_schema(node: dict) -> dict: + return {"version": "1.0", "extensions": {"nodes": [node]}} + + +def _schema_with_computed_attribute(computed_attribute: dict) -> dict: + schema = _valid_schema() + schema["nodes"][0]["attributes"][0]["computed_attribute"] = computed_attribute + return schema + + +def _schema_with_choices(choices: list[dict]) -> dict: + schema = _valid_schema() + schema["nodes"][0]["attributes"][0]["choices"] = choices + return schema + + +def _schema_with_parameters(parameters: dict) -> dict: + schema = _valid_schema() + schema["nodes"][0]["attributes"][0]["parameters"] = parameters + return schema + + +def _schema_with_kind_and_parameters(kind: str, parameters: dict) -> dict: + schema = _valid_schema() + schema["nodes"][0]["attributes"][0]["kind"] = kind + schema["nodes"][0]["attributes"][0]["parameters"] = parameters + return schema + + +def _relationship_out_of_enum(field: str, value: str) -> dict: + schema = _valid_schema() + schema["nodes"][0]["relationships"][0][field] = value + return schema + + +def _attribute_out_of_enum_kind(kind: str) -> dict: + schema = _valid_schema() + schema["nodes"][0]["attributes"][0]["kind"] = kind + return schema + + +def _schema_with_attribute_fields(**fields: object) -> dict: + schema = _valid_schema() + schema["nodes"][0]["attributes"][0].update(fields) + return schema + + +def _schema_with_relationship_fields(**fields: object) -> dict: + schema = _valid_schema() + schema["nodes"][0]["relationships"][0].update(fields) + return schema + + +def _schema_with_node_fields(**fields: object) -> dict: + schema = _valid_schema() + schema["nodes"][0].update(fields) + return schema + + +def _schema_with_generic_fields(**fields: object) -> dict: + schema = _valid_schema() + schema["generics"][0].update(fields) + return schema + + +def _schema_with_root_fields(**fields: object) -> dict: + schema = _valid_schema() + schema.update(fields) + return schema + + +def test_schema_root_models_are_importable_with_nodes_and_generics() -> None: + for root in (InfrahubSchemaWrite, InfrahubSchemaRead): + assert "nodes" in root.model_fields + assert "generics" in root.model_fields + + +def test_valid_payload_passes() -> None: + result = validate_schema(schema=_valid_schema()) + assert isinstance(result, SchemaValidationResult) + assert result.valid is True + assert result.errors == [] + # raise_for_status must be a no-op for a valid payload + result.raise_for_status() + + +def test_valid_payload_with_extensions_block_passes() -> None: + schema = _valid_schema() + schema["extensions"] = { + "nodes": [ + { + "kind": "InfraDevice", + "attributes": [{"name": "extra", "kind": "Text"}], + "relationships": [{"name": "peers", "peer": "InfraDevice", "cardinality": "many", "optional": True}], + } + ] + } + + result = validate_schema(schema=schema) + + assert result.valid is True, result.messages + + +def test_enum_backed_relationship_cardinality_valid_value_passes() -> None: + # A plain string for RelationshipCardinality is valid. + schema = _valid_schema() + schema["nodes"][0]["relationships"][0]["cardinality"] = "one" + + result = validate_schema(schema=schema) + + assert result.valid is True, result.messages + + +# --------------------------------------------------------------------------- +# Read-only fields are accepted with a warning +# --------------------------------------------------------------------------- + + +@dataclass +class ReadOnlyCase: + name: str + schema: dict + # Exact dotted paths expected among the reported warnings. + expected_fields: set[str] + + +READ_ONLY_CASES = [ + ReadOnlyCase( + name="attribute-inherited", + schema=_schema_with_attribute_fields(inherited=True), + expected_fields={"nodes[0].attributes[0].inherited"}, + ), + ReadOnlyCase( + name="relationship-inherited-and-hierarchical", + schema=_schema_with_relationship_fields(inherited=True, hierarchical="SomeGeneric"), + expected_fields={ + "nodes[0].relationships[0].inherited", + "nodes[0].relationships[0].hierarchical", + }, + ), + ReadOnlyCase( + name="generic-used-by", + schema=_schema_with_generic_fields(used_by=["InfraThing"]), + expected_fields={"generics[0].used_by"}, + ), + ReadOnlyCase( + name="node-hierarchy", + schema=_schema_with_node_fields(hierarchy="SomeGeneric"), + expected_fields={"nodes[0].hierarchy"}, + ), + ReadOnlyCase( + name="node-derived-kind-and-hash", + schema=_schema_with_node_fields(kind="InfraDevice", hash="abc123"), + expected_fields={"nodes[0].kind", "nodes[0].hash"}, + ), + ReadOnlyCase( + name="extension-attribute-inherited", + schema=_extension_node_schema( + {"kind": "InfraDevice", "attributes": [{"name": "extra", "kind": "Text", "inherited": True}]} + ), + expected_fields={"extensions.nodes[0].attributes[0].inherited"}, + ), + ReadOnlyCase( + name="root-keys-of-a-read-api-response", + schema=_schema_with_root_fields(main="abc123", profiles=[], templates=[], namespaces=[]), + expected_fields={"main", "profiles", "templates", "namespaces"}, + ), + # Every internal schema model carries `id` and `state`, so they appear on the nested value + # models of a schema dumped from those models even though they are not settable there. + ReadOnlyCase( + name="parameters-bookkeeping-fields", + schema=_schema_with_parameters({"min_length": 1, "id": None, "state": "present"}), + expected_fields={ + "nodes[0].attributes[0].parameters.id", + "nodes[0].attributes[0].parameters.state", + }, + ), + ReadOnlyCase( + name="choice-bookkeeping-fields", + schema=_schema_with_choices([{"name": "active", "id": None, "state": "present"}]), + expected_fields={ + "nodes[0].attributes[0].choices[0].id", + "nodes[0].attributes[0].choices[0].state", + }, + ), + # `transform` belongs to the TransformPython variant of the computed-attribute union, so it is + # known at this location but not settable on a Jinja2 one. + ReadOnlyCase( + name="computed-attribute-sibling-variant-field", + schema=_schema_with_computed_attribute({"kind": "Jinja2", "jinja2_template": "x", "transform": "t"}), + expected_fields={"nodes[0].attributes[0].computed_attribute.transform"}, + ), +] + + +@pytest.mark.parametrize("case", [pytest.param(tc, id=tc.name) for tc in READ_ONLY_CASES]) +def test_read_only_field_is_accepted_with_a_warning(case: ReadOnlyCase) -> None: + # A payload read back from Infrahub carries read-only fields, so it must still load; the user + # is told the value is ignored rather than having it dropped silently. + result = validate_schema(schema=case.schema) + + assert result.valid is True, result.messages + assert {warning.field for warning in result.warnings} == case.expected_fields + + +def test_read_only_warning_names_the_owning_kind_and_element() -> None: + # Consumers render a warning as kind + field rather than as a path, so the owning schema kind + # and the attribute/relationship carrying the field travel with the finding. + schema = _schema_with_attribute_fields(inherited=True) + + result = validate_schema(schema=schema) + + assert len(result.warnings) == 1 + warning = result.warnings[0] + assert warning.name == "inherited" + assert warning.kind == "InfraDevice" + assert warning.element == "hostname" + + +def test_nested_read_only_field_is_named_relative_to_its_owner() -> None: + # `id` is settable on an attribute but not on its nested parameters, so reporting the bare name + # would claim the wrong field is read-only -- and would collide with an `id` reported elsewhere. + schema = _schema_with_parameters({"id": None, "state": "present"}) + schema["extensions"] = {"nodes": [{"kind": "InfraDevice"}], "id": None} + + result = validate_schema(schema=schema) + + assert result.valid is True, result.messages + assert {warning.name for warning in result.warnings} == { + "parameters.id", + "parameters.state", + "extensions.id", + } + + +def test_read_only_fields_are_dropped_on_round_trip() -> None: + # A warning must not mean the value is kept: read-only fields are absent from the validated + # model, so they never reach the server. + schema = _valid_schema() + schema["nodes"][0]["hierarchy"] = "SomeGeneric" + schema["nodes"][0]["attributes"][0]["inherited"] = True + + assert validate_schema(schema=schema).valid is True + + dumped = InfrahubSchemaWrite.model_validate(schema).model_dump() + node = dumped["nodes"][0] + assert "hierarchy" not in node + assert "inherited" not in node["attributes"][0] + + +# --------------------------------------------------------------------------- +# Any other extra field is rejected +# --------------------------------------------------------------------------- + + +@dataclass +class UnknownFieldCase: + name: str + schema: dict + # Exact dotted paths expected among the reported error fields. + expected_fields: set[str] + + +UNKNOWN_FIELD_CASES = [ + UnknownFieldCase( + name="node-unknown-field", + schema=_schema_with_node_fields(not_a_field="boom"), + expected_fields={"nodes[0].not_a_field"}, + ), + UnknownFieldCase( + name="unknown-top-level-key", + schema=_schema_with_root_fields(not_a_root_field="boom"), + expected_fields={"not_a_root_field"}, + ), + UnknownFieldCase( + name="extension-attribute-unknown-field", + schema=_extension_node_schema( + {"kind": "InfraDevice", "attributes": [{"name": "extra", "kind": "Text", "not_a_field": "boom"}]} + ), + expected_fields={"extensions.nodes[0].attributes[0].not_a_field"}, + ), + UnknownFieldCase( + name="computed-attribute-unknown-field", + schema=_schema_with_computed_attribute({"kind": "Jinja2", "jinja2_template": "x", "not_a_real_field": "x"}), + expected_fields={"nodes[0].attributes[0].computed_attribute.not_a_real_field"}, + ), + UnknownFieldCase( + name="choice-unknown-field", + schema=_schema_with_choices([{"name": "active", "not_a_real_field": "x"}]), + expected_fields={"nodes[0].attributes[0].choices[0].not_a_real_field"}, + ), + UnknownFieldCase( + name="parameters-unknown-field", + schema=_schema_with_parameters({"not_a_real_param": 1}), + expected_fields={"nodes[0].attributes[0].parameters.not_a_real_param"}, + ), + # Parameters only valid for a different attribute kind do nothing on this one, so naming them + # is the only way the author learns the setting had no effect. + UnknownFieldCase( + name="number-attribute-number-pool-parameters", + schema=_schema_with_kind_and_parameters("Number", {"start_range": 1, "end_range": 9}), + expected_fields={ + "nodes[0].attributes[0].parameters.start_range", + "nodes[0].attributes[0].parameters.end_range", + }, + ), + UnknownFieldCase( + name="text-attribute-number-parameters", + schema=_schema_with_kind_and_parameters("Text", {"min_value": 1}), + expected_fields={"nodes[0].attributes[0].parameters.min_value"}, + ), + UnknownFieldCase( + name="generic-attribute-any-parameters", + schema=_schema_with_kind_and_parameters("Dropdown", {"regex": "x"}), + expected_fields={"nodes[0].attributes[0].parameters.regex"}, + ), +] + + +@pytest.mark.parametrize("case", [pytest.param(tc, id=tc.name) for tc in UNKNOWN_FIELD_CASES]) +def test_unknown_field_is_rejected_naming_the_field(case: UnknownFieldCase) -> None: + result = validate_schema(schema=case.schema) + + assert result.valid is False + assert _fields_named(result) == case.expected_fields + assert result.warnings == [] + + +def test_unknown_fields_are_reported_at_every_nesting_level_at_once() -> None: + # One pass must name every offending key rather than stopping at the first, so a payload is + # corrected in a single round. + schema = _valid_schema() + schema["not_a_root_field"] = "boom" + schema["nodes"][0]["not_a_node_field"] = "boom" + schema["nodes"][0]["attributes"][0]["not_an_attribute_field"] = "boom" + schema["nodes"][0]["relationships"][0]["not_a_relationship_field"] = "boom" + + result = validate_schema(schema=schema) + + assert result.valid is False + assert _fields_named(result) == { + "not_a_root_field", + "nodes[0].not_a_node_field", + "nodes[0].attributes[0].not_an_attribute_field", + "nodes[0].relationships[0].not_a_relationship_field", + } + + +def test_unknown_fields_are_not_reported_while_the_payload_is_otherwise_invalid() -> None: + # The validated model is what resolves the contract at each location, so a payload that fails + # validation reports that failure first and the extra fields once it is corrected. + schema = _schema_with_node_fields(not_a_field="boom") + schema["nodes"][0]["attributes"][0]["kind"] = "NotARealKind" + + result = validate_schema(schema=schema) + + assert result.valid is False + assert _fields_named(result) == {"nodes[0].attributes[0]"} + + +# --------------------------------------------------------------------------- +# Value violations are still rejected naming the field and the invalid value +# --------------------------------------------------------------------------- + + +@dataclass +class OutOfEnumCase: + name: str + schema: dict + # Exact dotted path expected among the reported error fields. For a discriminated union the + # unknown discriminator is reported against the container (attribute), not a leaf field. + expected_field: str + invalid_value: str + + +OUT_OF_ENUM_CASES = [ + OutOfEnumCase( + name="attribute-kind", + schema=_attribute_out_of_enum_kind("NotARealKind"), + expected_field="nodes[0].attributes[0]", + invalid_value="NotARealKind", + ), + OutOfEnumCase( + name="relationship-kind", + schema=_relationship_out_of_enum("kind", "NotARealKind"), + expected_field="nodes[0].relationships[0].kind", + invalid_value="NotARealKind", + ), + OutOfEnumCase( + name="relationship-cardinality", + schema=_relationship_out_of_enum("cardinality", "both"), + expected_field="nodes[0].relationships[0].cardinality", + invalid_value="both", + ), + OutOfEnumCase( + name="computed-attribute-kind", + schema=_schema_with_computed_attribute({"kind": "NotARealKind"}), + expected_field="nodes[0].attributes[0].Text.computed_attribute", + invalid_value="NotARealKind", + ), + OutOfEnumCase( + name="extension-attribute-kind", + schema=_extension_node_schema( + {"kind": "InfraDevice", "attributes": [{"name": "extra", "kind": "NotARealKind"}]} + ), + expected_field="extensions.nodes[0].attributes[0]", + invalid_value="NotARealKind", + ), + OutOfEnumCase( + name="extension-relationship-cardinality", + schema=_extension_node_schema( + {"kind": "InfraDevice", "relationships": [{"name": "peers", "peer": "InfraDevice", "cardinality": "both"}]} + ), + expected_field="extensions.nodes[0].relationships[0].cardinality", + invalid_value="both", + ), +] + + +@pytest.mark.parametrize("case", [pytest.param(tc, id=tc.name) for tc in OUT_OF_ENUM_CASES]) +def test_out_of_enum_value_is_rejected_naming_field_and_value(case: OutOfEnumCase) -> None: + result = validate_schema(schema=case.schema) + + assert result.valid is False + assert case.expected_field in _fields_named(result), result.messages + assert any(case.invalid_value in message for message in result.messages), result.messages + + +def test_enum_backed_relationship_cardinality_out_of_enum_value_is_rejected() -> None: + schema = _valid_schema() + schema["nodes"][0]["relationships"][0]["cardinality"] = "both" + + result = validate_schema(schema=schema) + + assert result.valid is False + assert any("cardinality" in message for message in result.messages), result.messages + + +def test_missing_version_is_rejected() -> None: + # The load endpoint requires ``version``, so a payload without it must be reported invalid + # offline too instead of passing here and being rejected on submission. + schema = _valid_schema() + del schema["version"] + + result = validate_schema(schema=schema) + + assert result.valid is False + assert _fields_named(result) == {"version"} + + +def test_raise_on_error_raises_value_error_naming_field() -> None: + # Exercises the raise_on_error path rather than the result verdict: an out-of-enum value must + # raise a ValueError naming the offending field. + schema = _valid_schema() + schema["nodes"][0]["attributes"][0]["kind"] = "NotARealKind" + + with pytest.raises(ValueError, match=r"kind"): + validate_schema(schema=schema, raise_on_error=True) + + +def test_valid_computed_attribute_block_passes() -> None: + result = validate_schema( + schema=_schema_with_computed_attribute({"kind": "Jinja2", "jinja2_template": "{{ name }}"}) + ) + + assert result.valid is True, result.messages + + +def test_computed_attribute_jinja2_without_template_is_rejected() -> None: + result = validate_schema(schema=_schema_with_computed_attribute({"kind": "Jinja2"})) + + assert result.valid is False + assert any("jinja2_template" in message for message in result.messages), result.messages + + +def test_computed_attribute_transform_python_without_transform_is_rejected() -> None: + result = validate_schema(schema=_schema_with_computed_attribute({"kind": "TransformPython"})) + + assert result.valid is False + assert any("transform" in message for message in result.messages), result.messages + + +def test_valid_choice_passes() -> None: + result = validate_schema(schema=_schema_with_choices([{"name": "active", "color": "#aabbcc", "label": "Active"}])) + + assert result.valid is True, result.messages + + +def test_choice_bad_color_is_rejected() -> None: + result = validate_schema(schema=_schema_with_choices([{"name": "active", "color": "not-a-color"}])) + + assert result.valid is False + assert any("color" in message for message in result.messages), result.messages + + +def test_valid_text_parameters_pass() -> None: + result = validate_schema(schema=_schema_with_parameters({"min_length": 1})) + + assert result.valid is True, result.messages + + +def test_number_attribute_accepts_number_parameters() -> None: + result = validate_schema(schema=_schema_with_kind_and_parameters("Number", {"min_value": 1})) + + assert result.valid is True, result.messages + + +def test_number_pool_attribute_accepts_number_pool_parameters() -> None: + result = validate_schema(schema=_schema_with_kind_and_parameters("NumberPool", {"start_range": 1, "end_range": 9})) + + assert result.valid is True, result.messages diff --git a/uv.lock b/uv.lock index d821a7b70..3fc930a07 100644 --- a/uv.lock +++ b/uv.lock @@ -676,7 +676,6 @@ wheels = [ [[package]] name = "infrahub-sdk" -version = "1.22.3" source = { editable = "." } dependencies = [ { name = "dulwich" }, @@ -701,6 +700,7 @@ all = [ { name = "pytest" }, { name = "pyyaml" }, { name = "rich" }, + { name = "ruamel-yaml" }, { name = "typer" }, ] ctl = [ @@ -713,6 +713,7 @@ ctl = [ { name = "pyarrow" }, { name = "pyyaml" }, { name = "rich" }, + { name = "ruamel-yaml" }, { name = "typer" }, ] @@ -724,6 +725,7 @@ dev = [ { name = "invoke" }, { name = "ipython", version = "8.37.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, { name = "ipython", version = "9.6.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, + { name = "jsonschema" }, { name = "mypy" }, { name = "prek" }, { name = "pytest" }, @@ -752,6 +754,7 @@ lint = [ ] tests = [ { name = "infrahub-testcontainers" }, + { name = "jsonschema" }, { name = "pytest" }, { name = "pytest-asyncio" }, { name = "pytest-clarity" }, @@ -791,13 +794,15 @@ requires-dist = [ { name = "pyyaml", marker = "extra == 'ctl'", specifier = ">=6" }, { name = "rich", marker = "extra == 'all'", specifier = ">=12,<14" }, { name = "rich", marker = "extra == 'ctl'", specifier = ">=12,<14" }, + { name = "ruamel-yaml", marker = "extra == 'all'", specifier = ">=0.18" }, + { name = "ruamel-yaml", marker = "extra == 'ctl'", specifier = ">=0.18" }, { name = "tomli", marker = "python_full_version < '3.11'", specifier = ">=1.1.0" }, { name = "typer", marker = "extra == 'all'", specifier = ">=0.15.0" }, { name = "typer", marker = "extra == 'ctl'", specifier = ">=0.15.0" }, { name = "ujson", specifier = ">=5" }, { name = "whenever", specifier = ">=0.9.3,<0.10.0" }, ] -provides-extras = ["ctl", "all"] +provides-extras = ["all", "ctl"] [package.metadata.requires-dev] dev = [ @@ -806,6 +811,7 @@ dev = [ { name = "infrahub-testcontainers", specifier = ">=1.7.3" }, { name = "invoke", specifier = ">=2.2.1" }, { name = "ipython" }, + { name = "jsonschema", specifier = ">=4.25.1" }, { name = "mypy", specifier = "==1.11.2" }, { name = "prek", specifier = ">=0.3.0" }, { name = "pytest", specifier = ">=9.0,<9.1" }, @@ -834,6 +840,7 @@ lint = [ ] tests = [ { name = "infrahub-testcontainers", specifier = ">=1.7.3" }, + { name = "jsonschema", specifier = ">=4.25.1" }, { name = "pytest", specifier = ">=9.0,<9.1" }, { name = "pytest-asyncio", specifier = ">=1.3,<1.4" }, { name = "pytest-clarity", specifier = ">=1.0.1" }, @@ -890,17 +897,17 @@ resolution-markers = [ "python_full_version < '3.11'", ] dependencies = [ - { name = "colorama", marker = "python_full_version < '3.11' and sys_platform == 'win32'" }, - { name = "decorator", marker = "python_full_version < '3.11'" }, - { name = "exceptiongroup", marker = "python_full_version < '3.11'" }, - { name = "jedi", marker = "python_full_version < '3.11'" }, - { name = "matplotlib-inline", marker = "python_full_version < '3.11'" }, - { name = "pexpect", marker = "python_full_version < '3.11' and sys_platform != 'emscripten' and sys_platform != 'win32'" }, - { name = "prompt-toolkit", marker = "python_full_version < '3.11'" }, - { name = "pygments", marker = "python_full_version < '3.11'" }, - { name = "stack-data", marker = "python_full_version < '3.11'" }, - { name = "traitlets", marker = "python_full_version < '3.11'" }, - { name = "typing-extensions", marker = "python_full_version < '3.11'" }, + { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "decorator" }, + { name = "exceptiongroup" }, + { name = "jedi" }, + { name = "matplotlib-inline" }, + { name = "pexpect", marker = "sys_platform != 'emscripten' and sys_platform != 'win32'" }, + { name = "prompt-toolkit" }, + { name = "pygments" }, + { name = "stack-data" }, + { name = "traitlets" }, + { name = "typing-extensions" }, ] sdist = { url = "https://files.pythonhosted.org/packages/85/31/10ac88f3357fc276dc8a64e8880c82e80e7459326ae1d0a211b40abf6665/ipython-8.37.0.tar.gz", hash = "sha256:ca815841e1a41a1e6b73a0b08f3038af9b2252564d01fc405356d34033012216", size = 5606088, upload-time = "2025-05-31T16:39:09.613Z" } wheels = [ @@ -917,17 +924,17 @@ resolution-markers = [ "python_full_version == '3.11.*'", ] dependencies = [ - { name = "colorama", marker = "python_full_version >= '3.11' and sys_platform == 'win32'" }, - { name = "decorator", marker = "python_full_version >= '3.11'" }, - { name = "ipython-pygments-lexers", marker = "python_full_version >= '3.11'" }, - { name = "jedi", marker = "python_full_version >= '3.11'" }, - { name = "matplotlib-inline", marker = "python_full_version >= '3.11'" }, - { name = "pexpect", marker = "python_full_version >= '3.11' and sys_platform != 'emscripten' and sys_platform != 'win32'" }, - { name = "prompt-toolkit", marker = "python_full_version >= '3.11'" }, - { name = "pygments", marker = "python_full_version >= '3.11'" }, - { name = "stack-data", marker = "python_full_version >= '3.11'" }, - { name = "traitlets", marker = "python_full_version >= '3.11'" }, - { name = "typing-extensions", marker = "python_full_version == '3.11.*'" }, + { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "decorator" }, + { name = "ipython-pygments-lexers" }, + { name = "jedi" }, + { name = "matplotlib-inline" }, + { name = "pexpect", marker = "sys_platform != 'emscripten' and sys_platform != 'win32'" }, + { name = "prompt-toolkit" }, + { name = "pygments" }, + { name = "stack-data" }, + { name = "traitlets" }, + { name = "typing-extensions", marker = "python_full_version < '3.12'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/2a/34/29b18c62e39ee2f7a6a3bba7efd952729d8aadd45ca17efc34453b717665/ipython-9.6.0.tar.gz", hash = "sha256:5603d6d5d356378be5043e69441a072b50a5b33b4503428c77b04cb8ce7bc731", size = 4396932, upload-time = "2025-09-29T10:55:53.948Z" } wheels = [ @@ -939,7 +946,7 @@ name = "ipython-pygments-lexers" version = "1.1.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "pygments", marker = "python_full_version >= '3.11'" }, + { name = "pygments" }, ] sdist = { url = "https://files.pythonhosted.org/packages/ef/4c/5dd1d8af08107f88c7f741ead7a40854b8ac24ddf9ae850afbcf698aa552/ipython_pygments_lexers-1.1.1.tar.gz", hash = "sha256:09c0138009e56b6854f9535736f4171d855c8c08a563a0dcd8022f78355c7e81", size = 8393, upload-time = "2025-01-17T11:24:34.505Z" } wheels = [ @@ -1472,8 +1479,8 @@ name = "pendulum" version = "3.1.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "python-dateutil", marker = "python_full_version < '3.13'" }, - { name = "tzdata", marker = "python_full_version < '3.13'" }, + { name = "python-dateutil" }, + { name = "tzdata" }, ] sdist = { url = "https://files.pythonhosted.org/packages/23/7c/009c12b86c7cc6c403aec80f8a4308598dfc5995e5c523a5491faaa3952e/pendulum-3.1.0.tar.gz", hash = "sha256:66f96303560f41d097bee7d2dc98ffca716fbb3a832c4b3062034c2d45865015", size = 85930, upload-time = "2025-04-19T14:30:01.675Z" } wheels = [