From 6541d2a02e1b8aa6f4bebb00b7302cdd02cb307e Mon Sep 17 00:00:00 2001 From: Wim Van Deun <7521270+wvandeun@users.noreply.github.com> Date: Wed, 19 Aug 2026 15:20:32 +0200 Subject: [PATCH 01/26] checkout python_sdk @v1.22.3 --- python_sdk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python_sdk b/python_sdk index 13f26b0a83..db6e0c7f63 160000 --- a/python_sdk +++ b/python_sdk @@ -1 +1 @@ -Subproject commit 13f26b0a83be50a16ef4009859ddd03fdf372d8a +Subproject commit db6e0c7f6352b03ca6e27c75da11192b4ef24c3d From 1958a2a5a33798f592f226f6c99465fa436217da Mon Sep 17 00:00:00 2001 From: Wim Van Deun <7521270+wvandeun@users.noreply.github.com> Date: Wed, 19 Aug 2026 15:26:00 +0200 Subject: [PATCH 02/26] release notes --- CHANGELOG.md | 7 +++++ ...te-rename-close-branch-owned-edge.fixed.md | 1 - changelog/9662.fixed.md | 1 - .../release-notes/infrahub/release-1_10_9.mdx | 27 +++++++++++++++++++ 4 files changed, 34 insertions(+), 2 deletions(-) delete mode 100644 changelog/+attribute-rename-close-branch-owned-edge.fixed.md delete mode 100644 changelog/9662.fixed.md create mode 100644 docs/docs/release-notes/infrahub/release-1_10_9.mdx diff --git a/CHANGELOG.md b/CHANGELOG.md index 45086005ce..9744c18d0e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,13 @@ This project uses [*towncrier*](https://towncrier.readthedocs.io/) and the chang +## [Infrahub - v1.10.9](https://github.com/opsmill/infrahub/tree/infrahub-v1.10.9) - 2026-08-19 + +### Fixed + +- Fixed a crash when loading the Tasks page where a task was tagged with a related node whose kind could no longer be resolved (for example a deleted definition or a stale tag). Such unresolvable related nodes are now omitted from the task instead of causing a GraphQL resolver error. ([#9662](https://github.com/opsmill/infrahub/issues/9662)) +- Renaming an attribute in a schema on a branch now correctly closes the old attribute instead of only opening a newer path to the new attribute. This issue would have been mostly invisible to the user unless an attribute was renamed on a user's branch and that branch was then rebased, in which case there could be duplicated paths to the new attribute which could result in unexpected behavior when updating its value. + ## [Infrahub - v1.10.8](https://github.com/opsmill/infrahub/tree/infrahub-v1.10.8) - 2026-08-14 ### Fixed diff --git a/changelog/+attribute-rename-close-branch-owned-edge.fixed.md b/changelog/+attribute-rename-close-branch-owned-edge.fixed.md deleted file mode 100644 index b371cb5e67..0000000000 --- a/changelog/+attribute-rename-close-branch-owned-edge.fixed.md +++ /dev/null @@ -1 +0,0 @@ -Renaming an attribute in a schema on a branch now correctly closes the old attribute instead of only opening a newer path to the new attribute. This issue would have been mostly invisible to the user unless an attribute was renamed on a user's branch and that branch was then rebased, in which case there could be duplicated paths to the new attribute which could result in unexpected behavior when updating its value. diff --git a/changelog/9662.fixed.md b/changelog/9662.fixed.md deleted file mode 100644 index 4969f24463..0000000000 --- a/changelog/9662.fixed.md +++ /dev/null @@ -1 +0,0 @@ -Fixed a crash when loading the Tasks page where a task was tagged with a related node whose kind could no longer be resolved (for example a deleted definition or a stale tag). Such unresolvable related nodes are now omitted from the task instead of causing a GraphQL resolver error. diff --git a/docs/docs/release-notes/infrahub/release-1_10_9.mdx b/docs/docs/release-notes/infrahub/release-1_10_9.mdx new file mode 100644 index 0000000000..b9d0d4f215 --- /dev/null +++ b/docs/docs/release-notes/infrahub/release-1_10_9.mdx @@ -0,0 +1,27 @@ +--- +title: Release 1.10.9 +release_date: 2026-08-19 +release_type: patch +description: "Fixes a crash on the Tasks page when a task references a related node whose kind can no longer be resolved, and an attribute rename on a branch leaving the old attribute open, which could duplicate paths to the renamed attribute after a rebase." +--- + + + + + + + + + + + + + + + +
Release Number1.10.9
Release DateAugust 19th, 2026
Tag[infrahub-v1.10.9](https://github.com/opsmill/infrahub/releases/tag/infrahub-v1.10.9)
+ +### Fixed + +- Fixed a crash when loading the Tasks page where a task was tagged with a related node whose kind could no longer be resolved (for example a deleted definition or a stale tag). Such unresolvable related nodes are now omitted from the task instead of causing a GraphQL resolver error. ([#9662](https://github.com/opsmill/infrahub/issues/9662)) +- Renaming an attribute in a schema on a branch now correctly closes the old attribute instead of only opening a newer path to the new attribute. This issue would have been mostly invisible to the user unless an attribute was renamed on a user's branch and that branch was then rebased, in which case there could be duplicated paths to the new attribute which could result in unexpected behavior when updating its value. From 122fa24a07ad30e241d3b0a7121c73c6196b8529 Mon Sep 17 00:00:00 2001 From: Wim Van Deun <7521270+wvandeun@users.noreply.github.com> Date: Wed, 19 Aug 2026 15:28:51 +0200 Subject: [PATCH 03/26] version bump to v1.10.9 --- pyproject.toml | 2 +- python_testcontainers/pyproject.toml | 2 +- python_testcontainers/uv.lock | 10 +++++----- uv.lock | 2 +- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 116757b5ab..5f11107cd3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "infrahub-server" -version = "1.10.8" +version = "1.10.9" description = "Infrahub is taking a new approach to Infrastructure Management by providing a new generation of datastore to organize and control all the data that defines how an infrastructure should run." authors = [{ name = "OpsMill", email = "info@opsmill.com" }] requires-python = ">=3.12,<3.15" diff --git a/python_testcontainers/pyproject.toml b/python_testcontainers/pyproject.toml index 2d52fd09bb..50a6759160 100644 --- a/python_testcontainers/pyproject.toml +++ b/python_testcontainers/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "infrahub-testcontainers" -version = "1.10.8" +version = "1.10.9" requires-python = ">=3.10" description = "Testcontainers instance for Infrahub to easily build integration tests" diff --git a/python_testcontainers/uv.lock b/python_testcontainers/uv.lock index f335f82ac9..1b4af6fc4d 100644 --- a/python_testcontainers/uv.lock +++ b/python_testcontainers/uv.lock @@ -521,7 +521,7 @@ wheels = [ [[package]] name = "infrahub-testcontainers" -version = "1.10.8" +version = "1.10.9" source = { editable = "." } dependencies = [ { name = "httpx" }, @@ -751,8 +751,8 @@ name = "pendulum" version = "3.1.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "python-dateutil" }, - { name = "tzdata" }, + { name = "python-dateutil", marker = "python_full_version < '3.13'" }, + { name = "tzdata", marker = "python_full_version < '3.13'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/23/7c/009c12b86c7cc6c403aec80f8a4308598dfc5995e5c523a5491faaa3952e/pendulum-3.1.0.tar.gz", hash = "sha256:66f96303560f41d097bee7d2dc98ffca716fbb3a832c4b3062034c2d45865015", size = 85930, upload-time = "2025-04-19T14:30:01.675Z" } wheels = [ @@ -1840,8 +1840,8 @@ name = "whenever" version = "0.9.3" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "tzdata", marker = "sys_platform == 'win32'" }, - { name = "tzlocal", marker = "sys_platform != 'darwin' and sys_platform != 'linux'" }, + { name = "tzdata", marker = "python_full_version >= '3.13' and sys_platform == 'win32'" }, + { name = "tzlocal", marker = "python_full_version >= '3.13' and sys_platform != 'darwin' and sys_platform != 'linux'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/87/ae/dcbfee50237cedca9153cac045dff6d93b81886f44f82c86493856592d55/whenever-0.9.3.tar.gz", hash = "sha256:22e6f8366767ab3c8be6d9e21a27bc56be2e50f0f2c66d78e8ee86497b579f1a", size = 256933, upload-time = "2025-10-16T19:44:40.832Z" } wheels = [ diff --git a/uv.lock b/uv.lock index 37c7942b30..23fa37272c 100644 --- a/uv.lock +++ b/uv.lock @@ -1429,7 +1429,7 @@ wheels = [ [[package]] name = "infrahub-server" -version = "1.10.8" +version = "1.10.9" source = { editable = "." } dependencies = [ { name = "aio-pika" }, From 85201d833e8ac2c6a3550f144bb73adb23baa708 Mon Sep 17 00:00:00 2001 From: opsmill-bot Date: Wed, 19 Aug 2026 14:13:34 +0000 Subject: [PATCH 04/26] chore: update docker-compose --- docker-compose.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index e6ca790152..7378a9c2a9 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -244,7 +244,7 @@ services: - 6362:6362 task-manager: - image: "${INFRAHUB_DOCKER_IMAGE:-registry.opsmill.io/opsmill/infrahub}:${VERSION:-1.10.8}" + image: "${INFRAHUB_DOCKER_IMAGE:-registry.opsmill.io/opsmill/infrahub}:${VERSION:-1.10.9}" command: uvicorn --host 0.0.0.0 --port 4200 --factory infrahub.prefect_server.app:create_infrahub_prefect restart: unless-stopped depends_on: @@ -277,7 +277,7 @@ services: retries: 5 infrahub-server: - image: "${INFRAHUB_DOCKER_IMAGE:-registry.opsmill.io/opsmill/infrahub}:${VERSION:-1.10.8}" + image: "${INFRAHUB_DOCKER_IMAGE:-registry.opsmill.io/opsmill/infrahub}:${VERSION:-1.10.9}" restart: unless-stopped command: > gunicorn --config backend/infrahub/serve/gunicorn_config.py @@ -323,7 +323,7 @@ services: deploy: mode: replicated replicas: 2 - image: "${INFRAHUB_DOCKER_IMAGE:-registry.opsmill.io/opsmill/infrahub}:${VERSION:-1.10.8}" + image: "${INFRAHUB_DOCKER_IMAGE:-registry.opsmill.io/opsmill/infrahub}:${VERSION:-1.10.9}" command: prefect worker start --type infrahubasync --pool infrahub-worker --with-healthcheck restart: unless-stopped depends_on: From 1e79bc2b516df9d8fc932f2291e8987634574100 Mon Sep 17 00:00:00 2001 From: Wim Van Deun <7521270+wvandeun@users.noreply.github.com> Date: Wed, 19 Aug 2026 20:23:41 +0200 Subject: [PATCH 05/26] checkout python_sdk @v1.23.0 --- python_sdk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python_sdk b/python_sdk index f9e28cfd59..99a380ac14 160000 --- a/python_sdk +++ b/python_sdk @@ -1 +1 @@ -Subproject commit f9e28cfd5958946759f113fd9fe29422adc8fcea +Subproject commit 99a380ac145cb549687bc2b8030cf5edf2f5a492 From 79760e09a9b86b69eb55aa161125e7ca49ec91cc Mon Sep 17 00:00:00 2001 From: Wim Van Deun <7521270+wvandeun@users.noreply.github.com> Date: Wed, 19 Aug 2026 23:28:47 +0200 Subject: [PATCH 06/26] fix version in docker compose file --- docker-compose.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 9ffb42f906..534e664caf 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -267,7 +267,7 @@ services: - 6362:6362 task-manager: - image: "${INFRAHUB_DOCKER_IMAGE:-registry.opsmill.io/opsmill/infrahub}:${VERSION:-1.10.9}" + image: "${INFRAHUB_DOCKER_IMAGE:-registry.opsmill.io/opsmill/infrahub}:${VERSION:-1.11.0}" command: uvicorn --host 0.0.0.0 --port 4200 --factory infrahub.prefect_server.app:create_infrahub_prefect restart: unless-stopped depends_on: @@ -300,7 +300,7 @@ services: retries: 5 infrahub-server: - image: "${INFRAHUB_DOCKER_IMAGE:-registry.opsmill.io/opsmill/infrahub}:${VERSION:-1.10.9}" + image: "${INFRAHUB_DOCKER_IMAGE:-registry.opsmill.io/opsmill/infrahub}:${VERSION:-1.11.0}" restart: unless-stopped command: > gunicorn --config backend/infrahub/serve/gunicorn_config.py @@ -346,7 +346,7 @@ services: deploy: mode: replicated replicas: 2 - image: "${INFRAHUB_DOCKER_IMAGE:-registry.opsmill.io/opsmill/infrahub}:${VERSION:-1.10.9}" + image: "${INFRAHUB_DOCKER_IMAGE:-registry.opsmill.io/opsmill/infrahub}:${VERSION:-1.11.0}" command: prefect worker start --type infrahubasync --pool infrahub-worker --with-healthcheck restart: unless-stopped depends_on: From 9868ac685be301d464680fdf569ebe361ab4e476 Mon Sep 17 00:00:00 2001 From: Infrahub Date: Wed, 19 Aug 2026 14:24:41 +0000 Subject: [PATCH 07/26] test: stop the cache and message-bus fixtures from leaking their config overrides MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `memory_cache` and `bus_simulator` each set a `config.OVERRIDE` field and never put it back. The `dependency_provider.scope(...)` unwinds on teardown, but `build_cache()` and `build_message_bus()` consult `config.OVERRIDE` *first*, so the override outlives the class that installed it and every later resolution in that xdist worker gets the previous class's throwaway adapter. For the cache that surfaces as ResourceNotFoundError: Diff summary for pipeline was not found in the cache in `TestProposedChange::test_run_generators_validate_requested_jobs`. The test writes the diff summary through a cache built from `config.SETTINGS.cache.driver` (Redis) and `run_generators` reads it back via `get_cache()`. Once the override leaks, the write goes to Redis and the read goes to the leftover MemoryCache. It reads as flaky but it is scheduling: it fails exactly when xdist puts `test_artifact_regen_e2e.py`, which uses `memory_cache`, on the same worker earlier in the session. Runs 32238651760 and 32153198842 had both files on gw3 and failed; run 32251034448 had them on gw3 and gw1 and passed. For the message bus nothing fails today — a stale BusSimulator swallows messages instead of raising — so it is fixed here before it costs a debugging session. Save and restore in a `finally`, matching the neighbouring `workflow_local` fixture and every other override site in the suite. Co-Authored-By: Claude Opus 5 (1M context) --- backend/tests/helpers/test_app.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/backend/tests/helpers/test_app.py b/backend/tests/helpers/test_app.py index c550b89130..b51f7e4663 100644 --- a/backend/tests/helpers/test_app.py +++ b/backend/tests/helpers/test_app.py @@ -96,20 +96,28 @@ async def bus_simulator( # Creating another service object to get service correctly initialized is a hack. # We should either reuse `service` fixture (leading to circular fixture dependencies issue atm), # or ideally properly patch production code responsible for Bus instantiation instead + original = config.OVERRIDE.message_bus bus = BusSimulator() _ = await InfrahubServices.new(database=db, workflow=WorkflowLocalExecution(), message_bus=bus) config.OVERRIDE.message_bus = bus - with dependency_provider.scope(build_message_bus, lambda: bus): - yield bus + try: + with dependency_provider.scope(build_message_bus, lambda: bus): + yield bus + finally: + config.OVERRIDE.message_bus = original @pytest.fixture(scope="class") async def memory_cache( self, db: InfrahubDatabase, dependency_provider: Provider ) -> AsyncGenerator[MemoryCache, None]: + original = config.OVERRIDE.cache cache = MemoryCache() config.OVERRIDE.cache = cache - with dependency_provider.scope(build_cache, lambda: cache): - yield cache + try: + with dependency_provider.scope(build_cache, lambda: cache): + yield cache + finally: + config.OVERRIDE.cache = original @pytest.fixture(scope="class") async def register_internal_schema(self, db: InfrahubDatabase, default_branch: Branch) -> SchemaBranch: From 9ce068665b97213dfd7f75728b71ed94c2cb9429 Mon Sep 17 00:00:00 2001 From: Infrahub Date: Wed, 19 Aug 2026 14:28:14 +0000 Subject: [PATCH 08/26] test: do not retry a failed Prefect task manager setup once per test class MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `setup_task_manager_once` recorded only success, so a Prefect test server that came up and then stopped responding was retried by every later test class in that xdist worker. The retry is not cheap. The setup does not fail fast against an unreachable server — it blocks on the API until the pytest timeout fires — so each retry cost the full 300s. In run 32238651760 that turned one broken worker into 46 `Failed: Timeout >300.0s` errors across five test files and pushed the session into its 1800s limit, with the original httpx.ReadTimeout buried under 45 identical copies. Remember the failure alongside the success and re-raise it, chained, on every later call. The worker still fails, but once, in seconds, with the cause attached to the first error rather than the forty-sixth. `except BaseException` is deliberate: the pytest timeout raises `Failed`, which does not derive from `Exception`, and that is exactly the failure worth remembering. The once-per-process state moves onto a `TaskManagerSetup` object that takes the setup callable as a constructor argument, so the tests drive it with recording and failing doubles instead of patching the module — the adapter pattern the testing guidelines ask for. `setup_task_manager_once()` keeps its signature and callers. Co-Authored-By: Claude Opus 5 (1M context) --- backend/tests/helpers/task_manager.py | 39 +++++++++-- .../tests/unit/helpers/test_task_manager.py | 70 +++++++++++++++++++ 2 files changed, 104 insertions(+), 5 deletions(-) create mode 100644 backend/tests/unit/helpers/test_task_manager.py diff --git a/backend/tests/helpers/task_manager.py b/backend/tests/helpers/task_manager.py index 60ae083975..13416d0a57 100644 --- a/backend/tests/helpers/task_manager.py +++ b/backend/tests/helpers/task_manager.py @@ -5,17 +5,46 @@ the calls are slow (several seconds of API round-trips), so fixtures should reuse a single setup per process instead of repeating it for every test or test class. +A failure is remembered the same way a success is. An unreachable Prefect test +server does not fail fast — the setup blocks until the pytest timeout fires — so +retrying it for every later test class costs that timeout each time and buries the +original cause under a wall of identical errors. + Tests that intentionally corrupt the shared task manager state must restore it themselves before yielding back, otherwise later tests will observe the corruption. """ +from collections.abc import Awaitable, Callable + from infrahub.workflows.initialization import setup_task_manager -_state = {"initialized": False} + +class TaskManagerSetup: + def __init__(self, setup: Callable[[], Awaitable[None]] = setup_task_manager) -> None: + self._setup = setup + self._initialized = False + self._failure: BaseException | None = None + + async def run_once(self) -> None: + if self._failure is not None: + raise RuntimeError("Prefect task manager setup already failed in this process") from self._failure + + if self._initialized: + return + + try: + await self._setup() + # The pytest timeout raises Failed, which derives from BaseException, and that is + # the failure worth remembering most. + except BaseException as exc: + self._failure = exc + raise + + self._initialized = True + + +_setup = TaskManagerSetup() async def setup_task_manager_once() -> None: - if _state["initialized"]: - return - await setup_task_manager() - _state["initialized"] = True + await _setup.run_once() diff --git a/backend/tests/unit/helpers/test_task_manager.py b/backend/tests/unit/helpers/test_task_manager.py new file mode 100644 index 0000000000..2ad537a933 --- /dev/null +++ b/backend/tests/unit/helpers/test_task_manager.py @@ -0,0 +1,70 @@ +import pytest + +from tests.helpers.task_manager import TaskManagerSetup + + +class TimeoutFailure(BaseException): + """Stands in for pytest's Failed, which derives from BaseException rather than Exception.""" + + +class RecordingSetup: + """Counts how many times the task manager setup was actually run.""" + + def __init__(self) -> None: + self.calls = 0 + + async def __call__(self) -> None: + self.calls += 1 + + +class FailingSetup(RecordingSetup): + """Stands in for a Prefect test server that accepts connections but never answers.""" + + def __init__(self, error: BaseException) -> None: + super().__init__() + self.error = error + + async def __call__(self) -> None: + await super().__call__() + raise self.error + + +async def test_setup_runs_once_across_repeated_calls() -> None: + setup = RecordingSetup() + once = TaskManagerSetup(setup=setup) + + await once.run_once() + await once.run_once() + await once.run_once() + + assert setup.calls == 1 + + +async def test_failed_setup_is_reported_without_being_rerun() -> None: + setup = FailingSetup(TimeoutError("prefect server is unreachable")) + once = TaskManagerSetup(setup=setup) + + with pytest.raises(TimeoutError, match=r"^prefect server is unreachable$"): + await once.run_once() + + for _ in range(3): + with pytest.raises( + RuntimeError, match=r"^Prefect task manager setup already failed in this process$" + ) as exc_info: + await once.run_once() + assert isinstance(exc_info.value.__cause__, TimeoutError) + + assert setup.calls == 1 + + +async def test_failure_that_bypasses_exception_is_remembered() -> None: + setup = FailingSetup(TimeoutFailure("Timeout >300.0s")) + once = TaskManagerSetup(setup=setup) + + with pytest.raises(TimeoutFailure, match=r"^Timeout >300\.0s$"): + await once.run_once() + + with pytest.raises(RuntimeError, match=r"^Prefect task manager setup already failed in this process$"): + await once.run_once() + + assert setup.calls == 1 From e9b4b6dcb8270cdf5273f7884525302f0e206d06 Mon Sep 17 00:00:00 2001 From: Aaron McCarty Date: Wed, 19 Aug 2026 17:19:30 -0500 Subject: [PATCH 09/26] fix(graphql): reject user-supplied branched_from on branch creation (#10306) * fix(graphql): reject user-supplied branched_from on branch creation BranchCreate persisted a client-supplied branched_from verbatim, letting a new branch appear to have been created in the past and expose data since deleted on all branches. The mutation now rejects the field with an error, and both branched_from and origin_branch (already restricted to the default branch) are marked deprecated on BranchCreateInput. Co-Authored-By: Claude Fable 5 * chore(frontend): revert spurious gql.tada cache reorder The @deprecated directives added to BranchCreateInput do not change any document type, so graphql-cache.d.ts should not have been touched. The committed version only reordered entries, which no longer matches what `pnpm codegen:graphql` emits, failing frontend-validate-graphql-types. Co-Authored-By: Claude Opus 5 (1M context) * fix(graphql): reject empty branched_from and raise ValidationError Addresses cubic review feedback on #10306. A supplied-but-falsy `branched_from` (empty string) slipped past the truthiness check and blew up later as `TimestampFormatError: Invalid time format for ` inside the branch-create flow. Check for presence of a non-null value instead, so every client-supplied value is rejected up front. An explicit `null` is still accepted: it is equivalent to omitting the field and rejecting it would break clients that serialize None. Raise `ValidationError` (HTTP 422, centrally handled) rather than a bare `ValueError`, matching how the rest of this mutation reports invalid input. Co-Authored-By: Claude Opus 5 (1M context) * fix(graphql): close the same falsy-input hole for origin_branch `origin_branch: ""` passed the truthiness guard and then overrode the "main" default on the model, producing a branch whose origin_branch is empty. Compare against a non-null value so any supplied value other than the default branch is rejected, and raise ValidationError to match how the neighbouring branched_from check reports invalid input. An explicit null on an optional field previously reached the pydantic model and failed to validate against a non-optional default (origin_branch, description, sync_with_git). Drop nulls when building the model so an explicit null behaves like omitting the field. Rework the BranchCreate input tests onto the dataclass parametrize pattern, covering both rejected fields and the omitted/null fallbacks. Co-Authored-By: Claude Opus 5 (1M context) * add removal version --------- Co-authored-by: Claude Fable 5 --- backend/infrahub/graphql/mutations/branch.py | 26 ++- .../graphql/mutations/test_branch.py | 154 ++++++++++++++++++ changelog/+block-user-branched-from.fixed.md | 1 + .../src/shared/api/graphql/generated/types.ts | 2 + schema/schema.graphql | 6 +- 5 files changed, 180 insertions(+), 9 deletions(-) create mode 100644 changelog/+block-user-branched-from.fixed.md diff --git a/backend/infrahub/graphql/mutations/branch.py b/backend/infrahub/graphql/mutations/branch.py index 9740a0ef11..04f345695a 100644 --- a/backend/infrahub/graphql/mutations/branch.py +++ b/backend/infrahub/graphql/mutations/branch.py @@ -49,10 +49,19 @@ class BranchCreateInput(InputObjectType): id = String(required=False) name = String(required=True) description = String(required=False) - origin_branch = String(required=False) - branched_from = String(required=False) + origin_branch = InputField( + String(required=False), + deprecation_reason="Branches can only be created from the default branch. Will be removed after version 1.12.", + ) + branched_from = InputField( + String(required=False), + deprecation_reason="branched_from is set by the server and cannot be provided. Will be removed after version 1.12.", + ) sync_with_git = Boolean(required=False) - is_isolated = InputField(Boolean(required=False), deprecation_reason="Non isolated mode is not supported anymore") + is_isolated = InputField( + Boolean(required=False), + deprecation_reason="Non-isolated mode is not supported anymore. Will be removed after version 1.12.", + ) class BranchCreate(Mutation): @@ -77,13 +86,18 @@ async def mutate( background_execution: bool = False, wait_until_completion: bool = True, ) -> Self: - if data.origin_branch and data.origin_branch != registry.default_branch: - raise ValueError(f"origin_branch must be '{registry.default_branch}'") + origin_branch = data.get("origin_branch") + if origin_branch is not None and origin_branch != registry.default_branch: + raise ValidationError(f"origin_branch must be '{registry.default_branch}'") + if data.get("branched_from") is not None: + raise ValidationError( + "branched_from input is deprecated and cannot be set, it will be the create time of the branch." + ) graphql_context: GraphqlContext = info.context task: dict | None = None - model = BranchCreateModel(**data) + model = BranchCreateModel(**{key: value for key, value in data.items() if value is not None}) await apply_external_context(graphql_context=graphql_context, context_input=context) try: diff --git a/backend/tests/component/graphql/mutations/test_branch.py b/backend/tests/component/graphql/mutations/test_branch.py index 3132b39780..726213303b 100644 --- a/backend/tests/component/graphql/mutations/test_branch.py +++ b/backend/tests/component/graphql/mutations/test_branch.py @@ -1,3 +1,4 @@ +from dataclasses import dataclass from typing import Any from unittest.mock import AsyncMock, patch @@ -19,6 +20,7 @@ from infrahub.core.schema.schema_branch import SchemaBranch from infrahub.core.timestamp import Timestamp from infrahub.database import InfrahubDatabase +from infrahub.exceptions import BranchNotFoundError from infrahub.graphql.initialization import prepare_graphql_params from infrahub.services import InfrahubServices from infrahub.services.adapters.workflow.local import WorkflowLocalExecution @@ -28,6 +30,158 @@ from tests.helpers.graphql import graphql, graphql_mutation from tests.helpers.test_app import TestInfrahubApp +BRANCH_CREATE = """ +mutation( + $name: String! + $description: String + $originBranch: String + $branchedFrom: String + $syncWithGit: Boolean +) { + BranchCreate( + data: { + name: $name + description: $description + origin_branch: $originBranch + branched_from: $branchedFrom + sync_with_git: $syncWithGit + } + ) { + ok + object { + id + name + description + origin_branch + branched_from + sync_with_git + } + } +} +""" + +BRANCHED_FROM_ERROR = "branched_from input is deprecated and cannot be set, it will be the create time of the branch." + + +@dataclass +class RejectedInputTestCase: + name: str + """Descriptive name for the test scenario.""" + + branch_name: str + """Name of the branch the mutation attempts to create.""" + + variables: dict[str, Any] + """Optional BranchCreate input variables sent alongside the branch name.""" + + expected_message: str + """The exact GraphQL error message the mutation must return.""" + + +REJECTED_INPUT_TEST_CASES: list[RejectedInputTestCase] = [ + RejectedInputTestCase( + name="branched_from_timestamp_rejected", + branch_name="own-branched-from", + variables={"branchedFrom": "2020-01-01T00:00:00.000Z"}, + expected_message=BRANCHED_FROM_ERROR, + ), + RejectedInputTestCase( + name="branched_from_empty_string_rejected", + branch_name="empty-branched-from", + variables={"branchedFrom": ""}, + expected_message=BRANCHED_FROM_ERROR, + ), + RejectedInputTestCase( + name="origin_branch_other_than_default_rejected", + branch_name="other-origin-branch", + variables={"originBranch": "not-the-default-branch"}, + expected_message="origin_branch must be 'main'", + ), + RejectedInputTestCase( + name="origin_branch_empty_string_rejected", + branch_name="empty-origin-branch", + variables={"originBranch": ""}, + expected_message="origin_branch must be 'main'", + ), +] + + +class TestBranchCreateInputValidation(TestInfrahubApp): + @pytest.mark.parametrize( + "test_case", + [pytest.param(tc, id=tc.name) for tc in REJECTED_INPUT_TEST_CASES], + ) + async def test_server_owned_input_is_rejected( + self, + db: InfrahubDatabase, + default_branch: Branch, + register_core_models_schema: SchemaBranch, + session_admin: AccountSession, + client: InfrahubClient, + service: InfrahubServices, + test_case: RejectedInputTestCase, + ) -> None: + """branched_from and origin_branch are decided by the server, so a client-supplied value is an input error.""" + result = await graphql_mutation( + query=BRANCH_CREATE, + db=db, + service=service, + branch=default_branch, + account_session=session_admin, + variables={"name": test_case.branch_name} | test_case.variables, + ) + + assert result.errors is not None + assert len(result.errors) == 1 + assert result.errors[0].message == test_case.expected_message + + with pytest.raises(BranchNotFoundError): + await Branch.get_by_name(db=db, name=test_case.branch_name) + + @pytest.mark.parametrize( + ("branch_name", "variables"), + [ + pytest.param("omitted-optional-input", {}, id="optional_input_omitted"), + pytest.param( + "null-optional-input", + {"description": None, "originBranch": None, "branchedFrom": None, "syncWithGit": None}, + id="optional_input_explicitly_null", + ), + ], + ) + async def test_unset_optional_input_falls_back_to_defaults( + self, + db: InfrahubDatabase, + default_branch: Branch, + register_core_models_schema: SchemaBranch, + session_admin: AccountSession, + client: InfrahubClient, + service: InfrahubServices, + branch_name: str, + variables: dict[str, Any], + ) -> None: + """An explicit null says no more than omitting the field, so both must land on the server defaults.""" + result = await graphql_mutation( + query=BRANCH_CREATE, + db=db, + service=service, + branch=default_branch, + account_session=session_admin, + variables={"name": branch_name} | variables, + ) + + assert result.errors is None + assert result.data + assert result.data["BranchCreate"]["ok"] is True + + branch = await Branch.get_by_name(db=db, name=branch_name) + assert isinstance(branch.description, str) + assert not branch.description + assert branch.origin_branch == default_branch.name + assert branch.sync_with_git is True + assert isinstance(branch.branched_from, str) + assert branch.branched_from + class TestBranchCreate(TestInfrahubApp): async def test_branch_create( diff --git a/changelog/+block-user-branched-from.fixed.md b/changelog/+block-user-branched-from.fixed.md new file mode 100644 index 0000000000..80ad8511e7 --- /dev/null +++ b/changelog/+block-user-branched-from.fixed.md @@ -0,0 +1 @@ +The `BranchCreate` GraphQL mutation now rejects a client-supplied `branched_from` value with an error, and the field is marked deprecated. `branched_from` is an internal field managed by the application. The `origin_branch` field, which was already rejected for any value other than the default branch, is now also marked deprecated. An empty string is now rejected for either field instead of being silently accepted, and an explicit `null` on an optional field is treated the same as omitting it. diff --git a/frontend/app/src/shared/api/graphql/generated/types.ts b/frontend/app/src/shared/api/graphql/generated/types.ts index 13b61aed64..422fcb3dd9 100644 --- a/frontend/app/src/shared/api/graphql/generated/types.ts +++ b/frontend/app/src/shared/api/graphql/generated/types.ts @@ -296,12 +296,14 @@ export type BranchCreate = { }; export type BranchCreateInput = { + /** @deprecated branched_from is set by the server and cannot be provided */ branched_from?: InputMaybe; description?: InputMaybe; id?: InputMaybe; /** @deprecated Non isolated mode is not supported anymore */ is_isolated?: InputMaybe; name: Scalars['String']['input']; + /** @deprecated Branches can only be created from the default branch */ origin_branch?: InputMaybe; sync_with_git?: InputMaybe; }; diff --git a/schema/schema.graphql b/schema/schema.graphql index 62714f5a12..14ec8f883a 100644 --- a/schema/schema.graphql +++ b/schema/schema.graphql @@ -253,12 +253,12 @@ type BranchCreate { } input BranchCreateInput { - branched_from: String + branched_from: String @deprecated(reason: "branched_from is set by the server and cannot be provided. Will be removed after version 1.12.") description: String id: String - is_isolated: Boolean @deprecated(reason: "Non isolated mode is not supported anymore") + is_isolated: Boolean @deprecated(reason: "Non-isolated mode is not supported anymore. Will be removed after version 1.12.") name: String! - origin_branch: String + origin_branch: String @deprecated(reason: "Branches can only be created from the default branch. Will be removed after version 1.12.") sync_with_git: Boolean } From 39fa69612837607f259831def3147eb8b092ab5b Mon Sep 17 00:00:00 2001 From: Infrahub Date: Wed, 19 Aug 2026 15:08:13 +0000 Subject: [PATCH 10/26] perf(testcontainers): replace rabbitmq-diagnostics healthcheck with raw TCP probe Every rabbitmq-diagnostics invocation boots a full Erlang VM, costing ~2.1s of CPU time per check (measured on rabbitmq:4.2.1-management). At the 1s healthcheck interval this pegs more than two cores per broker container for the entire life of the stack, and the ~0.3-1s wall time races against the 1s timeout on loaded CI machines. check_port_connectivity only verifies that the listener ports accept TCP connections, so a bash /dev/tcp probe on 5672 provides the identical readiness signal at ~1ms per check, with no broker log noise (RabbitMQ does not log connections closed before the protocol header). Exec form is required because the image's /bin/sh (dash) lacks /dev/tcp support. Co-Authored-By: Claude Fable 5 --- .../infrahub_testcontainers/docker-compose-cluster.test.yml | 5 ++++- .../infrahub_testcontainers/docker-compose.test.yml | 5 ++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/python_testcontainers/infrahub_testcontainers/docker-compose-cluster.test.yml b/python_testcontainers/infrahub_testcontainers/docker-compose-cluster.test.yml index 9801b98d85..58f5ce12ec 100644 --- a/python_testcontainers/infrahub_testcontainers/docker-compose-cluster.test.yml +++ b/python_testcontainers/infrahub_testcontainers/docker-compose-cluster.test.yml @@ -38,7 +38,10 @@ services: RABBITMQ_DEFAULT_USER: infrahub RABBITMQ_DEFAULT_PASS: infrahub healthcheck: - test: rabbitmq-diagnostics -q check_port_connectivity + # raw TCP probe instead of rabbitmq-diagnostics: each diagnostics call + # boots a full Erlang VM (~2s CPU), which at a 1s interval pegs two + # cores per broker for the life of the stack + test: ["CMD", "bash", "-c", " Date: Wed, 19 Aug 2026 09:20:32 +0000 Subject: [PATCH 11/26] test: stop the webhook traceback fixture from reconfiguring logging process-wide The configured_logging fixture called configure_logging(production=False, log_level="DEBUG"). That routine is application startup code: it sets the root log level, replaces the root handler and reconfigures structlog, and undoes none of it. Called per test it left the root logger at DEBUG for the rest of the xdist worker, overriding the WARNING level pytest_configure pins, so every later test in that worker logged a line per Bolt message from the Neo4j driver. Install only what the assertions need instead: extract the filter installation from configure_logging as install_traceback_suppression_filter, call that from the fixture and remove the filter after the yield. The fixture is renamed traceback_suppression_installed to say what it now does. Also record the general rule in the backend testing guidelines and the Python testing agent rules: leave process-global state as you found it, and never call an application startup routine from a test. Co-Authored-By: Claude Opus 5 (1M context) --- .agents/rules/testing-python.md | 4 +++ backend/infrahub/log.py | 25 +++++++++---- .../webhook/test_traceback_suppression.py | 35 ++++++++++++++----- dev/guidelines/backend/testing.md | 27 ++++++++++++++ 4 files changed, 75 insertions(+), 16 deletions(-) diff --git a/.agents/rules/testing-python.md b/.agents/rules/testing-python.md index 71b02a1db3..86b873c218 100644 --- a/.agents/rules/testing-python.md +++ b/.agents/rules/testing-python.md @@ -63,6 +63,10 @@ Skip tests that only exercise library behavior: plain `Enum` value/round-trip ch If the logic needs only in-memory inputs (a `SchemaBranch`, a dataclass, a pure function), write a unit test without DB fixtures — don't default to a component test because a neighbor uses one. Use the database or containers only when behavior genuinely depends on them. +## Don't leak process-global state + +Every test in an xdist worker shares one interpreter. Change `logging` levels/handlers/filters, `structlog` config, module-level registries/singletons, `sys.path`/`sys.modules` or env vars only through a save/restore fixture (change it, `yield`, restore it), or `monkeypatch` where it applies. Never call an application startup routine such as `infrahub.log.configure_logging` from a test — it owns the whole process and undoes nothing, so it reconfigures every later test in the worker. Install only the piece under test and remove it after the `yield`. See `dev/guidelines/backend/testing.md` §"Leave process-global state as you found it". + ## Test file placement Test files mirror source structure: `infrahub/core/node.py` → `tests/unit/core/test_node.py` diff --git a/backend/infrahub/log.py b/backend/infrahub/log.py index e93a54053a..563d7e1d8a 100644 --- a/backend/infrahub/log.py +++ b/backend/infrahub/log.py @@ -59,6 +59,22 @@ def filter(self, record: logging.LogRecord) -> bool: return type(exception) not in self._suppressed_types +def install_traceback_suppression_filter() -> TracebackSuppressionFilter: + """Install the traceback suppression filter on the Prefect run loggers and return it. + + Prefect ships flow/task run logs to its API; drop tracebacks for failures that are reported as a + clean classified reason rather than a crash to debug. The filter reads the shared registry that + each expected-failure type opts into via suppress_traceback_in_logs. + + The installed filter is returned so a caller that must leave logging state as it found it can + remove it again from every logger in PREFECT_RUN_LOGGERS. + """ + traceback_filter = TracebackSuppressionFilter(_TRACEBACK_SUPPRESSED_TYPES) + for prefect_logger_name in PREFECT_RUN_LOGGERS: + logging.getLogger(prefect_logger_name).addFilter(traceback_filter) + return traceback_filter + + def clear_log_context() -> None: structlog.contextvars.clear_contextvars() @@ -86,13 +102,8 @@ def configure_logging(production: bool, log_level: str) -> None: # the infrahub logger importlib.import_module("prefect.main") - # Prefect ships flow/task run logs to its API; drop tracebacks for failures that - # are reported as a clean classified reason rather than a crash to debug. Installed after the - # prefect.main import above so it survives Prefect's logging reset; reads the shared registry that - # each expected-failure type opts into via suppress_traceback_in_logs. - traceback_filter = TracebackSuppressionFilter(_TRACEBACK_SUPPRESSED_TYPES) - for prefect_logger_name in PREFECT_RUN_LOGGERS: - logging.getLogger(prefect_logger_name).addFilter(traceback_filter) + # Installed after the prefect.main import above so it survives Prefect's logging reset. + install_traceback_suppression_filter() shared_processors: list[Processor] = [ structlog.contextvars.merge_contextvars, diff --git a/backend/tests/component/webhook/test_traceback_suppression.py b/backend/tests/component/webhook/test_traceback_suppression.py index 701708b659..6ee295d027 100644 --- a/backend/tests/component/webhook/test_traceback_suppression.py +++ b/backend/tests/component/webhook/test_traceback_suppression.py @@ -1,12 +1,13 @@ from __future__ import annotations import logging +from typing import TYPE_CHECKING import httpx import pytest from prefect import flow, task -from infrahub.log import configure_logging +from infrahub.log import PREFECT_RUN_LOGGERS, install_traceback_suppression_filter from infrahub.webhook.classifier import ( EXPECTED_DELIVERY_ERRORS, ClassifiedFailure, @@ -15,6 +16,9 @@ WebhookFailureClassifier, ) +if TYPE_CHECKING: + from collections.abc import Generator + CLASSIFIED_MESSAGE = "The target responded with HTTP 404." @@ -46,12 +50,25 @@ async def _send_classifying_in_task() -> None: @pytest.fixture -def configured_logging() -> None: - # Register the traceback filter on the Prefect run loggers, as production startup does. - configure_logging(production=False, log_level="DEBUG") - - -async def test_classified_failure_logs_no_traceback(configured_logging: None, caplog: pytest.LogCaptureFixture) -> None: +def traceback_suppression_installed() -> Generator[None, None, None]: + """Register the traceback filter on the Prefect run loggers, as production startup does, then remove it. + + Only the filter is installed, not the whole of configure_logging: that startup routine also raises + the root log level, replaces the root handler and reconfigures structlog, none of which these + assertions need and none of which it undoes. Called from a fixture it would leak that state into + every test that follows in the same worker — overriding the WARNING root level the suite pins in + pytest_configure, so unrelated tests drown in DEBUG records from the database driver and the HTTP + client. + """ + traceback_filter = install_traceback_suppression_filter() + yield + for prefect_logger_name in PREFECT_RUN_LOGGERS: + logging.getLogger(prefect_logger_name).removeFilter(traceback_filter) + + +async def test_classified_failure_logs_no_traceback( + traceback_suppression_installed: None, caplog: pytest.LogCaptureFixture +) -> None: with ( caplog.at_level(logging.INFO, logger="prefect.flow_runs"), pytest.raises(WebhookDeliveryError, match=r"^The target responded with HTTP 404\.$"), @@ -66,7 +83,7 @@ async def test_classified_failure_logs_no_traceback(configured_logging: None, ca async def test_classified_failure_from_task_logs_no_traceback( - configured_logging: None, caplog: pytest.LogCaptureFixture + traceback_suppression_installed: None, caplog: pytest.LogCaptureFixture ) -> None: # The transport error is caught and classified inside the task, so the failure the engine records # for the task run is a delivery error whose traceback is dropped — not the raw transport stacktrace. @@ -84,7 +101,7 @@ async def test_classified_failure_from_task_logs_no_traceback( async def test_unclassified_failure_logs_a_traceback( - configured_logging: None, caplog: pytest.LogCaptureFixture + traceback_suppression_installed: None, caplog: pytest.LogCaptureFixture ) -> None: with ( caplog.at_level(logging.INFO, logger="prefect.flow_runs"), diff --git a/dev/guidelines/backend/testing.md b/dev/guidelines/backend/testing.md index 74ce53aba8..6c4ce93216 100644 --- a/dev/guidelines/backend/testing.md +++ b/dev/guidelines/backend/testing.md @@ -111,6 +111,33 @@ The module provides individual node/generic schemas (`CAR`, `DEVICE`, `TAG`, `PE `config.SETTINGS` is populated from `INFRAHUB_*` environment variables at process start, so values exported in the developer's shell leak into the test process. Any test whose behavior depends on a settings field must pin it in a save/restore fixture (set the value, `yield`, restore the original) — see `import_every_remote_branch` in `backend/tests/integration/git/conftest.py`. Never assume a field holds its default. +## Leave process-global state as you found it + +Under `pytest-xdist` every test in a worker shares one interpreter, so whatever a test changes outside +its own fixtures stays changed for every test that follows it there. Touch global state only through a +save/restore fixture (change it, `yield`, restore the original). Pinning a setting, above, is one case +of that rule; it also covers: + +- the `logging` module — root and per-logger levels, handlers, filters +- `structlog` configuration +- module-level registries, caches and singletons +- environment variables (prefer `monkeypatch.setenv`, which restores on teardown) +- `sys.path`, `sys.modules`, warning filters + +**Never call an application startup routine from a test.** `infrahub.log.configure_logging` is the +example to learn from: it runs once at process start and owns the process when it does — setting the +root log level, replacing the root handler and reconfiguring structlog — so, being startup code, it has +no counterpart that undoes any of that. Called from a fixture it silently reconfigures every later test +in the worker. Install only the piece the test needs, extracting it from the startup routine when it is +not already reusable, and undo it after the `yield` — see `traceback_suppression_installed` in +`backend/tests/component/webhook/test_traceback_suppression.py`, which installs the traceback +suppression filter alone rather than calling `configure_logging`. + +Such a leak is invisible locally and expensive in CI. A root logger left at `DEBUG` overrides the +`WARNING` level `pytest_configure` pins, and the Neo4j driver then logs a line per Bolt message for +every test that follows in that worker: one job produced 185k lines of driver output and pushed three +unrelated tests past their 300s timeout. + ## Dataclass Test Case Pattern For parametrized tests with multiple scenarios, use dataclasses to define test cases. This pattern provides type safety, readable test IDs, and clear separation between test data and test logic. From b0d02d87bd20d5da837f014f11373148d40fd8c8 Mon Sep 17 00:00:00 2001 From: Infrahub Date: Wed, 19 Aug 2026 09:37:31 +0000 Subject: [PATCH 12/26] test(webhook): guard that the traceback suppression fixture restores logging state MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The three suppression tests pass whether the fixture installs only the filter or calls configure_logging, since that routine installs the same filter — nothing committed distinguished them, so the leak could come back unnoticed. Extract the fixture body into a _traceback_suppression context manager and assert its contract directly: the Prefect run loggers carry exactly the filter it installed while it is open, and both their filter lists and the root log level are back to their previous values once it closes. Driving the context manager rather than probing state from a later test module keeps the guard hermetic — no dependence on collection order or on which xdist worker picks the file up. Co-Authored-By: Claude Opus 5 (1M context) --- .../webhook/test_traceback_suppression.py | 51 +++++++++++++++---- 1 file changed, 42 insertions(+), 9 deletions(-) diff --git a/backend/tests/component/webhook/test_traceback_suppression.py b/backend/tests/component/webhook/test_traceback_suppression.py index 6ee295d027..bc2e38004e 100644 --- a/backend/tests/component/webhook/test_traceback_suppression.py +++ b/backend/tests/component/webhook/test_traceback_suppression.py @@ -1,13 +1,14 @@ from __future__ import annotations import logging +from contextlib import contextmanager from typing import TYPE_CHECKING import httpx import pytest from prefect import flow, task -from infrahub.log import PREFECT_RUN_LOGGERS, install_traceback_suppression_filter +from infrahub.log import PREFECT_RUN_LOGGERS, TracebackSuppressionFilter, install_traceback_suppression_filter from infrahub.webhook.classifier import ( EXPECTED_DELIVERY_ERRORS, ClassifiedFailure, @@ -17,7 +18,7 @@ ) if TYPE_CHECKING: - from collections.abc import Generator + from collections.abc import Generator, Iterator, Sequence CLASSIFIED_MESSAGE = "The target responded with HTTP 404." @@ -49,21 +50,29 @@ async def _send_classifying_in_task() -> None: await _classify_transport_failure() -@pytest.fixture -def traceback_suppression_installed() -> Generator[None, None, None]: +@contextmanager +def _traceback_suppression() -> Iterator[TracebackSuppressionFilter]: """Register the traceback filter on the Prefect run loggers, as production startup does, then remove it. Only the filter is installed, not the whole of configure_logging: that startup routine also raises the root log level, replaces the root handler and reconfigures structlog, none of which these - assertions need and none of which it undoes. Called from a fixture it would leak that state into - every test that follows in the same worker — overriding the WARNING root level the suite pins in + assertions need and none of which it undoes. Called per test it would leak that state into every + test that follows in the same worker — overriding the WARNING root level the suite pins in pytest_configure, so unrelated tests drown in DEBUG records from the database driver and the HTTP client. """ traceback_filter = install_traceback_suppression_filter() - yield - for prefect_logger_name in PREFECT_RUN_LOGGERS: - logging.getLogger(prefect_logger_name).removeFilter(traceback_filter) + try: + yield traceback_filter + finally: + for prefect_logger_name in PREFECT_RUN_LOGGERS: + logging.getLogger(prefect_logger_name).removeFilter(traceback_filter) + + +@pytest.fixture +def traceback_suppression_installed() -> Generator[None, None, None]: + with _traceback_suppression(): + yield async def test_classified_failure_logs_no_traceback( @@ -112,3 +121,27 @@ async def test_unclassified_failure_logs_a_traceback( logged_exceptions = [record.exc_info[1] for record in caplog.records if record.exc_info] assert [type(exc) for exc in logged_exceptions] == [RuntimeError] assert str(logged_exceptions[0]) == "boom" + + +def _run_logger_filters() -> dict[str, Sequence[object]]: + # Logger.filters is a union of filter forms; the identity of what is attached is all that matters here. + return {name: list(logging.getLogger(name).filters) for name in PREFECT_RUN_LOGGERS} + + +def test_traceback_suppression_leaves_logging_state_unchanged() -> None: + """The suppression context must hand logging back exactly as it found it. + + The assertions above cannot tell this apart from calling configure_logging, which installs the same + filter — only the process-wide state that routine also changes, and never restores, separates them. + """ + root_logger = logging.getLogger() + level_before, filters_before = root_logger.level, _run_logger_filters() + + with _traceback_suppression() as traceback_filter: + assert _run_logger_filters() == { + name: [*filters_before[name], traceback_filter] for name in PREFECT_RUN_LOGGERS + } + assert root_logger.level == level_before + + assert _run_logger_filters() == filters_before + assert root_logger.level == level_before From f3c7a851d75092f0d28615620d42aa215b3dc331 Mon Sep 17 00:00:00 2001 From: Infrahub Date: Wed, 19 Aug 2026 09:38:31 +0000 Subject: [PATCH 13/26] test(log): cover that startup installs the traceback filter on the run loggers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Extracting install_traceback_suppression_filter left the configure_logging call site uncovered: dropping it would keep every traceback suppression test passing, since they install the filter themselves. Assert instead on what importing infrahub.log already did — its module-level configure_logging call is the production wiring — so the call site is covered without a test reconfiguring logging for the rest of the worker. Co-Authored-By: Claude Opus 5 (1M context) --- backend/tests/unit/test_log.py | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/backend/tests/unit/test_log.py b/backend/tests/unit/test_log.py index cbe2ad9663..a792cc85a5 100644 --- a/backend/tests/unit/test_log.py +++ b/backend/tests/unit/test_log.py @@ -2,7 +2,12 @@ import logging -from infrahub.log import _TRACEBACK_SUPPRESSED_TYPES, TracebackSuppressionFilter, suppress_traceback_in_logs +from infrahub.log import ( + _TRACEBACK_SUPPRESSED_TYPES, + PREFECT_RUN_LOGGERS, + TracebackSuppressionFilter, + suppress_traceback_in_logs, +) from infrahub.webhook.classifier import ClassifiedFailure, StatusClass, WebhookDeliveryError @@ -42,3 +47,17 @@ class _ExpectedFailureError(Exception): ... # The production filter is wired to this shared registry, so a decorated type is suppressed. assert TracebackSuppressionFilter(_TRACEBACK_SUPPRESSED_TYPES).filter(_record(_ExpectedFailureError())) is False + + +def test_startup_installs_the_filter_on_the_prefect_run_loggers() -> None: + """Importing infrahub.log configures logging for the process, which is what installs the filter. + + Asserting on that import rather than calling configure_logging again keeps the wiring covered + without a test reconfiguring logging for every test that follows it in the worker. + """ + installed_on = [ + name + for name in PREFECT_RUN_LOGGERS + if any(isinstance(log_filter, TracebackSuppressionFilter) for log_filter in logging.getLogger(name).filters) + ] + assert installed_on == list(PREFECT_RUN_LOGGERS) From 4dff0fe49850604282cb1ec33a30178c331b8615 Mon Sep 17 00:00:00 2001 From: Infrahub Date: Wed, 19 Aug 2026 21:55:07 +0000 Subject: [PATCH 14/26] test(log): cut the rejected alternative from the startup test docstring A docstring should say what the test asserts, not what it deliberately does not do. The paragraph explaining why the test reads the state left by the import rather than calling configure_logging again is a note on a choice already made; the rule it follows is written down in the backend testing guidelines. The first line still explains why asserting on import-time state is the wiring assertion, which is the part a reader needs. Co-Authored-By: Claude Opus 5 (1M context) --- backend/tests/unit/test_log.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/backend/tests/unit/test_log.py b/backend/tests/unit/test_log.py index a792cc85a5..0e5e8611d6 100644 --- a/backend/tests/unit/test_log.py +++ b/backend/tests/unit/test_log.py @@ -50,11 +50,7 @@ class _ExpectedFailureError(Exception): ... def test_startup_installs_the_filter_on_the_prefect_run_loggers() -> None: - """Importing infrahub.log configures logging for the process, which is what installs the filter. - - Asserting on that import rather than calling configure_logging again keeps the wiring covered - without a test reconfiguring logging for every test that follows it in the worker. - """ + """Importing infrahub.log configures logging for the process, which is what installs the filter.""" installed_on = [ name for name in PREFECT_RUN_LOGGERS From 928f911a39b55c8883a3fca3bc944f1feb24bb10 Mon Sep 17 00:00:00 2001 From: Infrahub Date: Wed, 19 Aug 2026 21:55:15 +0000 Subject: [PATCH 15/26] test(log): move the logging-state guard into the unit suite MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit test_traceback_suppression_leaves_logging_state_unchanged drives a context manager and reads logging state — it needs no database, no Prefect flow run and none of the component suite's fixtures, so it belongs next to the other infrahub.log tests in the unit suite. Move the install/remove cycle it drives to tests/helpers/log.py as traceback_suppression, so the webhook fixture and the guard share one implementation across suites and any future fixture that installs the filter inherits the guard. Its docstring keeps the line that says what it does; the paragraph on why it installs the filter alone rather than calling configure_logging is the rule the testing guidelines now carry, which the guidelines point at the helper for. Co-Authored-By: Claude Opus 5 (1M context) --- .../webhook/test_traceback_suppression.py | 50 ++----------------- backend/tests/helpers/log.py | 25 ++++++++++ backend/tests/unit/test_log.py | 25 ++++++++++ dev/guidelines/backend/testing.md | 4 +- 4 files changed, 55 insertions(+), 49 deletions(-) create mode 100644 backend/tests/helpers/log.py diff --git a/backend/tests/component/webhook/test_traceback_suppression.py b/backend/tests/component/webhook/test_traceback_suppression.py index bc2e38004e..06fcb850b0 100644 --- a/backend/tests/component/webhook/test_traceback_suppression.py +++ b/backend/tests/component/webhook/test_traceback_suppression.py @@ -1,14 +1,12 @@ from __future__ import annotations import logging -from contextlib import contextmanager from typing import TYPE_CHECKING import httpx import pytest from prefect import flow, task -from infrahub.log import PREFECT_RUN_LOGGERS, TracebackSuppressionFilter, install_traceback_suppression_filter from infrahub.webhook.classifier import ( EXPECTED_DELIVERY_ERRORS, ClassifiedFailure, @@ -16,9 +14,10 @@ WebhookDeliveryError, WebhookFailureClassifier, ) +from tests.helpers.log import traceback_suppression if TYPE_CHECKING: - from collections.abc import Generator, Iterator, Sequence + from collections.abc import Generator CLASSIFIED_MESSAGE = "The target responded with HTTP 404." @@ -50,28 +49,9 @@ async def _send_classifying_in_task() -> None: await _classify_transport_failure() -@contextmanager -def _traceback_suppression() -> Iterator[TracebackSuppressionFilter]: - """Register the traceback filter on the Prefect run loggers, as production startup does, then remove it. - - Only the filter is installed, not the whole of configure_logging: that startup routine also raises - the root log level, replaces the root handler and reconfigures structlog, none of which these - assertions need and none of which it undoes. Called per test it would leak that state into every - test that follows in the same worker — overriding the WARNING root level the suite pins in - pytest_configure, so unrelated tests drown in DEBUG records from the database driver and the HTTP - client. - """ - traceback_filter = install_traceback_suppression_filter() - try: - yield traceback_filter - finally: - for prefect_logger_name in PREFECT_RUN_LOGGERS: - logging.getLogger(prefect_logger_name).removeFilter(traceback_filter) - - @pytest.fixture def traceback_suppression_installed() -> Generator[None, None, None]: - with _traceback_suppression(): + with traceback_suppression(): yield @@ -121,27 +101,3 @@ async def test_unclassified_failure_logs_a_traceback( logged_exceptions = [record.exc_info[1] for record in caplog.records if record.exc_info] assert [type(exc) for exc in logged_exceptions] == [RuntimeError] assert str(logged_exceptions[0]) == "boom" - - -def _run_logger_filters() -> dict[str, Sequence[object]]: - # Logger.filters is a union of filter forms; the identity of what is attached is all that matters here. - return {name: list(logging.getLogger(name).filters) for name in PREFECT_RUN_LOGGERS} - - -def test_traceback_suppression_leaves_logging_state_unchanged() -> None: - """The suppression context must hand logging back exactly as it found it. - - The assertions above cannot tell this apart from calling configure_logging, which installs the same - filter — only the process-wide state that routine also changes, and never restores, separates them. - """ - root_logger = logging.getLogger() - level_before, filters_before = root_logger.level, _run_logger_filters() - - with _traceback_suppression() as traceback_filter: - assert _run_logger_filters() == { - name: [*filters_before[name], traceback_filter] for name in PREFECT_RUN_LOGGERS - } - assert root_logger.level == level_before - - assert _run_logger_filters() == filters_before - assert root_logger.level == level_before diff --git a/backend/tests/helpers/log.py b/backend/tests/helpers/log.py new file mode 100644 index 0000000000..448e894d30 --- /dev/null +++ b/backend/tests/helpers/log.py @@ -0,0 +1,25 @@ +"""Install the infrahub.log traceback suppression filter for a test, then remove it again.""" + +from __future__ import annotations + +import logging +from contextlib import contextmanager +from typing import TYPE_CHECKING + +from infrahub.log import PREFECT_RUN_LOGGERS, install_traceback_suppression_filter + +if TYPE_CHECKING: + from collections.abc import Iterator + + from infrahub.log import TracebackSuppressionFilter + + +@contextmanager +def traceback_suppression() -> Iterator[TracebackSuppressionFilter]: + """Register the traceback filter on the Prefect run loggers, as production startup does, then remove it.""" + traceback_filter = install_traceback_suppression_filter() + try: + yield traceback_filter + finally: + for prefect_logger_name in PREFECT_RUN_LOGGERS: + logging.getLogger(prefect_logger_name).removeFilter(traceback_filter) diff --git a/backend/tests/unit/test_log.py b/backend/tests/unit/test_log.py index 0e5e8611d6..a01c74b052 100644 --- a/backend/tests/unit/test_log.py +++ b/backend/tests/unit/test_log.py @@ -1,6 +1,7 @@ from __future__ import annotations import logging +from typing import TYPE_CHECKING from infrahub.log import ( _TRACEBACK_SUPPRESSED_TYPES, @@ -9,6 +10,10 @@ suppress_traceback_in_logs, ) from infrahub.webhook.classifier import ClassifiedFailure, StatusClass, WebhookDeliveryError +from tests.helpers.log import traceback_suppression + +if TYPE_CHECKING: + from collections.abc import Sequence def _record(exception: BaseException | None) -> logging.LogRecord: @@ -57,3 +62,23 @@ def test_startup_installs_the_filter_on_the_prefect_run_loggers() -> None: if any(isinstance(log_filter, TracebackSuppressionFilter) for log_filter in logging.getLogger(name).filters) ] assert installed_on == list(PREFECT_RUN_LOGGERS) + + +def _run_logger_filters() -> dict[str, Sequence[object]]: + # Logger.filters is a union of filter forms; the identity of what is attached is all that matters here. + return {name: list(logging.getLogger(name).filters) for name in PREFECT_RUN_LOGGERS} + + +def test_traceback_suppression_leaves_logging_state_unchanged() -> None: + """The suppression context must hand logging back exactly as it found it.""" + root_logger = logging.getLogger() + level_before, filters_before = root_logger.level, _run_logger_filters() + + with traceback_suppression() as traceback_filter: + assert _run_logger_filters() == { + name: [*filters_before[name], traceback_filter] for name in PREFECT_RUN_LOGGERS + } + assert root_logger.level == level_before + + assert _run_logger_filters() == filters_before + assert root_logger.level == level_before diff --git a/dev/guidelines/backend/testing.md b/dev/guidelines/backend/testing.md index 6c4ce93216..a2e7327114 100644 --- a/dev/guidelines/backend/testing.md +++ b/dev/guidelines/backend/testing.md @@ -129,8 +129,8 @@ example to learn from: it runs once at process start and owns the process when i root log level, replacing the root handler and reconfiguring structlog — so, being startup code, it has no counterpart that undoes any of that. Called from a fixture it silently reconfigures every later test in the worker. Install only the piece the test needs, extracting it from the startup routine when it is -not already reusable, and undo it after the `yield` — see `traceback_suppression_installed` in -`backend/tests/component/webhook/test_traceback_suppression.py`, which installs the traceback +not already reusable, and undo it after the `yield` — see `traceback_suppression` in +`backend/tests/helpers/log.py`, which the webhook suppression tests use to install the traceback suppression filter alone rather than calling `configure_logging`. Such a leak is invisible locally and expensive in CI. A root logger left at `DEBUG` overrides the From 7abb3fe30dd59e59e84338072e58fdfa69102ddc Mon Sep 17 00:00:00 2001 From: Infrahub Date: Wed, 19 Aug 2026 15:11:37 +0000 Subject: [PATCH 16/26] test(webhook): identify the render test's own flow run instead of counting `read_flow_runs()` returns at most 200 rows (the Prefect API's default page size, and the API rejects a larger limit), so once the session's webhook-process runs fill that page the render test's before/after count comparison saturates at 200 and can never be true again. That makes the test fail deterministically whenever `TestWebhookConfigure` lands on the same xdist worker, which is how five unrelated PRs went red between Aug 13 and Aug 18 with a message blaming server-side parameter rendering. Identify the run by its id and webhook instead, and assert the rendered parameters really are the plain strings the test exists to guard. Delete the webhook automations a test class registers at its teardown: the Prefect server is session-scoped, so a surviving all-branches automation turned every event emitted by every later test in the worker into a scheduled webhook-process run that no worker ever executes. Measured over the webhook package plus one event-heavy ipam class: 216-243 leftover runs and 233-235s without the cleanup, 1 run and 182-201s with it. Co-Authored-By: Claude Opus 5 (1M context) --- backend/tests/functional/webhook/conftest.py | 20 ++++++++++ .../tests/functional/webhook/test_render.py | 37 ++++++++++++++++--- dev/knowledge/backend/testing.md | 14 +++++++ 3 files changed, 66 insertions(+), 5 deletions(-) diff --git a/backend/tests/functional/webhook/conftest.py b/backend/tests/functional/webhook/conftest.py index c0940a99e2..584ec3e441 100644 --- a/backend/tests/functional/webhook/conftest.py +++ b/backend/tests/functional/webhook/conftest.py @@ -12,6 +12,9 @@ from infrahub.core.node import Node from infrahub.events.models import EventBranchContext, EventContext from infrahub.task_manager.flow_run.prefect_client import PrefectClientAdapter +from infrahub.trigger.constants import NAME_SEPARATOR +from infrahub.trigger.models import TriggerType +from infrahub.trigger.setup import gather_all_automations from infrahub.webhook.tasks import process from infrahub.workflows.catalogue import ( WEBHOOK_CONFIGURE, @@ -101,6 +104,23 @@ async def prefect_client(prefect_test_fixture: None) -> AsyncGenerator[PrefectCl yield client +@pytest.fixture(scope="class", autouse=True) +async def delete_webhook_automations(prefect_client: PrefectClient) -> AsyncGenerator[None, None]: + """Delete the webhook automations a test class registered, once the class is done with them. + + The Prefect test server is session-scoped, so an automation outlives the class that created it + while the webhook node behind it is dropped with the class database. A surviving all-branches + automation then turns every event any later test emits in this session into a scheduled + webhook-process run that no worker ever executes, filling the server's database and pushing the + run count past the API's 200-row page size. + """ + yield + webhook_prefix = f"{TriggerType.WEBHOOK.value}{NAME_SEPARATOR}" + for automation in await gather_all_automations(client=prefect_client): + if automation.id and automation.name.startswith(webhook_prefix): + await prefect_client.delete_automation(automation_id=automation.id) + + @pytest.fixture(scope="class") def flow_run_querier(prefect_client: PrefectClient) -> FlowRunQuerying: """A read-only view of the Prefect client, exposing only flow-run querying to tests.""" diff --git a/backend/tests/functional/webhook/test_render.py b/backend/tests/functional/webhook/test_render.py index 9216e9e17d..11627ee8cf 100644 --- a/backend/tests/functional/webhook/test_render.py +++ b/backend/tests/functional/webhook/test_render.py @@ -6,6 +6,7 @@ from uuid import uuid4 from prefect.client.schemas.filters import DeploymentFilter, DeploymentFilterId +from prefect.client.schemas.sorting import FlowRunSort from prefect.events.schemas.events import Event, Resource from prefect.types import DateTime @@ -19,6 +20,7 @@ if TYPE_CHECKING: from infrahub_sdk import InfrahubClient from prefect.client.orchestration import PrefectClient + from prefect.client.schemas.objects import FlowRun from infrahub.database import InfrahubDatabase @@ -63,7 +65,22 @@ async def test_branchless_event_triggers_webhook_process( deployment = await prefect_client.read_deployment_by_name(f"{WEBHOOK_PROCESS.name}/{WEBHOOK_PROCESS.name}") deployment_filter = DeploymentFilter(id=DeploymentFilterId(any_=[deployment.id])) - runs_before = len(await prefect_client.read_flow_runs(deployment_filter=deployment_filter)) + + async def read_process_runs() -> list[FlowRun]: + # Earlier tests leave webhook-process runs behind and a read is capped at the server's + # 200-row page size, so run counts saturate and only a run's identity is a usable signal. + return await prefect_client.read_flow_runs( + deployment_filter=deployment_filter, sort=FlowRunSort.EXPECTED_START_TIME_DESC + ) + + runs_before = {run.id for run in await read_process_runs()} + + async def read_new_runs() -> list[FlowRun]: + return [ + run + for run in await read_process_runs() + if run.id not in runs_before and run.parameters.get("webhook_id") == webhook.id + ] # A branch-less event: the resource carries no infrahub.branch.name, the id is a UUID and the # occurred time a datetime -- all values the action parameters must render as plain strings. @@ -76,10 +93,20 @@ async def test_branchless_event_triggers_webhook_process( ) await prefect_client._client.post("/events", json=[event.model_dump(mode="json")]) - runs_after = runs_before + new_runs: list[FlowRun] = [] for _ in range(PREFECT_EVENT_WAIT_SECONDS): - runs_after = len(await prefect_client.read_flow_runs(deployment_filter=deployment_filter)) - if runs_after > runs_before: + new_runs = await read_new_runs() + if new_runs: break await asyncio.sleep(1) - assert runs_after > runs_before, "webhook-process deployment was not run; server-side parameter render failed" + assert new_runs, "webhook-process deployment was not run; server-side parameter render failed" + + # Every value the deployment receives has to be a plain string, an absent branch included. + parameters = new_runs[0].parameters + assert parameters["event_id"] == str(event.id) + assert parameters["event_type"] == "infrahub.node.created" + assert parameters["event_occured_at"] == "2026-01-01 00:00:00+00:00" + branch_name = parameters["branch_name"] + assert isinstance(branch_name, str) + assert not branch_name + assert parameters["event_payload"] == {"data": {"node_id": "abc"}, "context": {}} diff --git a/dev/knowledge/backend/testing.md b/dev/knowledge/backend/testing.md index b5d0bae462..046ec5ebfd 100644 --- a/dev/knowledge/backend/testing.md +++ b/dev/knowledge/backend/testing.md @@ -418,6 +418,20 @@ async def test_logs_warning(caplog: pytest.LogCaptureFixture) -> None: This matches the pattern used in `test_webhook_header.py` and `test_models.py`. +### Prefect Server State Outlives the Test Class + +The Prefect test server is session-scoped — one per xdist worker — while the database and the +fixtures that populate it are class-scoped, so whatever a class registers on that server survives +it. Two rules follow: + +- Delete the automations a class created at its teardown. A surviving all-branches webhook + automation turns every event any later test emits into a scheduled flow run — no worker runs in + the functional suite, so nothing executes them — filling the server's SQLite database. +- Never assert on a flow-run count. `read_flow_runs()` returns at most `PREFECT_API_DEFAULT_LIMIT` + (200) rows and the API rejects a larger `limit`, so once that page is full a before/after + comparison saturates and can never be true again. Read newest-first + (`FlowRunSort.EXPECTED_START_TIME_DESC`) and identify the run by its id or parameters instead. + ### Functional Tests with `TestInfrahubApp` `TestInfrahubApp` provides a `memory_cache` fixture (class-scoped) that injects a `MemoryCache` via `dependency_provider.scope(build_cache, ...)`. Use it in functional tests to pre-fill and assert on cache state: From 20fad944ec48cb4af45a82aafc4ea9ebd148990d Mon Sep 17 00:00:00 2001 From: Pete Crocker Date: Thu, 20 Aug 2026 08:19:11 +0000 Subject: [PATCH 17/26] fix(ci): retry cosign transparency-log writes and upload SBOMs first MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The v1.10.7 release failed in publish-docker-image / sbom (run 31521732131) when `cosign attest` could not write to the Sigstore transparency log: Post "https://rekor.sigstore.dev/api/v1/log/entries": giving up after 2 attempt(s) Rekor was only briefly unavailable — the sign job had succeeded against the same infrastructure five minutes earlier, and the identical sbom job passed on the next nightly run with no code change. But cosign retries a failed log write only twice internally and exposes no knob to raise that, so a blip that short was enough to fail the release. The fallout was disproportionate: the SBOM upload step never ran, so v1.10.7 is the only release since v1.10.0 with no SBOM artifacts, and repository-dispatch (which needs publish-docker-image) was skipped, so downstream repos were never notified of the release. Three changes: - Wrap cosign sign and both cosign attest calls in a retry() helper: five attempts 60 seconds apart, covering a ~4-minute outage. The helper takes the command as arguments ("$@", no string/eval layer) and is pasted verbatim into the sign and sbom jobs — they run on separate runners, so it cannot be shared without a checkout or a third-party action in the signing path. - Upload the SBOM artifacts before attesting them, so a transparency-log outage can no longer cost us the SBOMs themselves. Attestation failures still fail the job, which is the correct outcome for a supply-chain step. - Set overwrite: true on the SBOM upload. Artifacts are scoped to the run rather than the attempt, so with the upload now preceding a step that can fail, a re-run would otherwise die at the upload step because the artifact name already exists from the earlier attempt. Every caller passes a version unique to its invocation, so the only artifact this can replace is the same SBOM from a previous attempt of the same run. Verified by extracting both run scripts from the parsed workflow and executing them against a stubbed cosign: immediate success (1 call, no sleeps), recovery after 3 failures (exit 0, 3 sleeps), budget exhaustion (5 calls, 4 sleeps, exit 1), and a terminally failing first attest stops the script before the second attest runs. The two retry() bodies are asserted byte-identical, and actionlint passes. Co-Authored-By: Claude Fable 5 --- .github/workflows/ci-docker-image.yml | 86 ++++++++++++++++++++++----- 1 file changed, 70 insertions(+), 16 deletions(-) diff --git a/.github/workflows/ci-docker-image.yml b/.github/workflows/ci-docker-image.yml index 7d07688feb..6d36a94e80 100644 --- a/.github/workflows/ci-docker-image.yml +++ b/.github/workflows/ci-docker-image.yml @@ -202,9 +202,33 @@ jobs: password: ${{ secrets.HARBOR_PASSWORD }} - name: Sign manifest + env: + IMAGE: ${{ vars.HARBOR_HOST }}/${{ github.repository }}@${{ needs.merge.outputs.digest }} run: | - cosign sign --yes --recursive --new-bundle-format=false --use-signing-config=false \ - "${{ vars.HARBOR_HOST }}/${{ github.repository }}@${{ needs.merge.outputs.digest }}" + # cosign gives up after two internal attempts when a Sigstore transparency + # log write fails, so a brief rekor.sigstore.dev blip is enough to fail a + # release. Retry the whole command instead; five attempts 60s apart cover a + # ~4-minute outage. The sbom job carries a verbatim copy of this function: + # the jobs run on separate runners, so sharing it would take a checkout or a + # third-party action in the signing path. Keep the two copies identical. + retry() { + local attempt + for attempt in 1 2 3 4 5; do + if "$@"; then + return 0 + fi + if [ "${attempt}" -lt 5 ]; then + echo "::warning::${1} ${2} failed (attempt ${attempt}/5), retrying in 60s" + sleep 60 + fi + done + echo "::error::${1} ${2} failed after 5 attempts" + return 1 + } + + retry cosign sign --yes --recursive \ + --new-bundle-format=false --use-signing-config=false \ + "${IMAGE}" sbom: needs: merge @@ -236,20 +260,16 @@ jobs: "${{ vars.HARBOR_HOST }}/${{ github.repository }}@${{ needs.merge.outputs.digest }}" \ --output cyclonedx-json=infrahub-sbom.cdx.json - - name: Attest SBOM (SPDX) - run: | - cosign attest --yes \ - --type spdxjson \ - --predicate infrahub-sbom.spdx.json \ - "${{ vars.HARBOR_HOST }}/${{ github.repository }}@${{ needs.merge.outputs.digest }}" - - - name: Attest SBOM (CycloneDX) - run: | - cosign attest --yes \ - --type cyclonedx \ - --predicate infrahub-sbom.cdx.json \ - "${{ vars.HARBOR_HOST }}/${{ github.repository }}@${{ needs.merge.outputs.digest }}" - + # Uploaded before the attestations so that a transparency-log outage cannot cost + # us the SBOMs themselves: v1.10.7 shipped without any because the attest step + # failed first and skipped this one. + # + # overwrite is required precisely because this now runs before a step that can + # fail. Artifacts are scoped to the run rather than the attempt, so on a re-run + # this name already exists from the earlier attempt and the default overwrite: + # false would fail the upload, breaking the recovery path this ordering exists to + # protect. Every caller passes a version unique to its invocation, so the only + # artifact this can replace is the same SBOM from a previous attempt. - name: Upload SBOM artifacts uses: actions/upload-artifact@v7 with: @@ -258,3 +278,37 @@ jobs: infrahub-sbom.spdx.json infrahub-sbom.cdx.json retention-days: 90 + overwrite: true + + - name: Attest SBOMs + env: + IMAGE: ${{ vars.HARBOR_HOST }}/${{ github.repository }}@${{ needs.merge.outputs.digest }} + run: | + # Same transparency-log flakiness the sign job guards against; this is a + # verbatim copy of that job's retry function (separate runners, so it cannot + # be shared without a checkout or a third-party action in the signing path). + # Keep the two copies identical. + retry() { + local attempt + for attempt in 1 2 3 4 5; do + if "$@"; then + return 0 + fi + if [ "${attempt}" -lt 5 ]; then + echo "::warning::${1} ${2} failed (attempt ${attempt}/5), retrying in 60s" + sleep 60 + fi + done + echo "::error::${1} ${2} failed after 5 attempts" + return 1 + } + + retry cosign attest --yes \ + --type spdxjson \ + --predicate infrahub-sbom.spdx.json \ + "${IMAGE}" + + retry cosign attest --yes \ + --type cyclonedx \ + --predicate infrahub-sbom.cdx.json \ + "${IMAGE}" From 7c1c161442cec82df6f9cfa20bd7b862a82d47c6 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 20 Aug 2026 11:48:51 +0300 Subject: [PATCH 18/26] chore(deps): bump the uv group across 2 directories with 5 updates (#10336) Bumps the uv group with 5 updates in the / directory: | Package | From | To | | --- | --- | --- | | [pydantic-settings](https://github.com/pydantic/pydantic-settings) | `2.14.1` | `2.14.2` | | [python-multipart](https://github.com/Kludex/python-multipart) | `0.0.27` | `0.0.31` | | [pyjwt](https://github.com/jpadilla/pyjwt) | `2.12.1` | `2.13.0` | | [cryptography](https://github.com/pyca/cryptography) | `48.0.0` | `50.0.0` | | [starlette](https://github.com/Kludex/starlette) | `1.2.1` | `1.3.1` | Bumps the uv group with 2 updates in the /python_testcontainers directory: [pydantic-settings](https://github.com/pydantic/pydantic-settings) and [starlette](https://github.com/Kludex/starlette). Updates `pydantic-settings` from 2.14.1 to 2.14.2 - [Release notes](https://github.com/pydantic/pydantic-settings/releases) - [Commits](https://github.com/pydantic/pydantic-settings/compare/v2.14.1...v2.14.2) Updates `python-multipart` from 0.0.27 to 0.0.31 - [Release notes](https://github.com/Kludex/python-multipart/releases) - [Changelog](https://github.com/Kludex/python-multipart/blob/main/CHANGELOG.md) - [Commits](https://github.com/Kludex/python-multipart/compare/0.0.27...0.0.31) Updates `pyjwt` from 2.12.1 to 2.13.0 - [Release notes](https://github.com/jpadilla/pyjwt/releases) - [Changelog](https://github.com/jpadilla/pyjwt/blob/master/CHANGELOG.rst) - [Commits](https://github.com/jpadilla/pyjwt/compare/2.12.1...2.13.0) Updates `cryptography` from 48.0.0 to 50.0.0 - [Changelog](https://github.com/pyca/cryptography/blob/main/CHANGELOG.rst) - [Commits](https://github.com/pyca/cryptography/compare/48.0.0...50.0.0) Updates `starlette` from 1.2.1 to 1.3.1 - [Release notes](https://github.com/Kludex/starlette/releases) - [Changelog](https://github.com/Kludex/starlette/blob/main/docs/release-notes.md) - [Commits](https://github.com/Kludex/starlette/compare/1.2.1...1.3.1) Updates `pydantic-settings` from 2.12.0 to 2.14.2 - [Release notes](https://github.com/pydantic/pydantic-settings/releases) - [Commits](https://github.com/pydantic/pydantic-settings/compare/v2.14.1...v2.14.2) Updates `starlette` from 1.2.1 to 1.3.1 - [Release notes](https://github.com/Kludex/starlette/releases) - [Changelog](https://github.com/Kludex/starlette/blob/main/docs/release-notes.md) - [Commits](https://github.com/Kludex/starlette/compare/1.2.1...1.3.1) --- updated-dependencies: - dependency-name: pydantic-settings dependency-version: 2.14.2 dependency-type: direct:production dependency-group: uv - dependency-name: python-multipart dependency-version: 0.0.31 dependency-type: direct:production dependency-group: uv - dependency-name: pyjwt dependency-version: 2.13.0 dependency-type: direct:production dependency-group: uv - dependency-name: cryptography dependency-version: 50.0.0 dependency-type: indirect dependency-group: uv - dependency-name: starlette dependency-version: 1.3.1 dependency-type: indirect dependency-group: uv - dependency-name: pydantic-settings dependency-version: 2.14.2 dependency-type: indirect dependency-group: uv - dependency-name: starlette dependency-version: 1.3.1 dependency-type: indirect dependency-group: uv ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pyproject.toml | 4 +- python_testcontainers/uv.lock | 20 +++--- uv.lock | 119 +++++++++++++++++----------------- 3 files changed, 70 insertions(+), 73 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 5954f3e660..1d63780d5e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -45,10 +45,10 @@ dependencies = [ "lunr>=0.7.0.post1,<0.8", "starlette-exporter>=0.23,<0.24", "prometheus-client>=0.25,<0.26", - "python-multipart==0.0.27", # Required by FastAPI to upload large files + "python-multipart==0.0.31", # Required by FastAPI to upload large files "asgi-correlation-id==4.2.0", # Middleware for FastAPI to generate ID per request "bcrypt>=4.1,<4.2", # Used to hash and validate password - "pyjwt==2.12.1", # Used to manage JWT tokens + "pyjwt==2.13.0", # Used to manage JWT tokens "uvicorn[standard]>=0.32,<0.33", "opentelemetry-instrumentation-aio-pika==0.65b0", "opentelemetry-instrumentation-fastapi==0.65b0", diff --git a/python_testcontainers/uv.lock b/python_testcontainers/uv.lock index cd4358e1e7..32f67108f0 100644 --- a/python_testcontainers/uv.lock +++ b/python_testcontainers/uv.lock @@ -750,8 +750,8 @@ name = "pendulum" version = "3.1.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "python-dateutil", marker = "python_full_version < '3.13'" }, - { name = "tzdata", marker = "python_full_version < '3.13'" }, + { name = "python-dateutil" }, + { name = "tzdata" }, ] sdist = { url = "https://files.pythonhosted.org/packages/23/7c/009c12b86c7cc6c403aec80f8a4308598dfc5995e5c523a5491faaa3952e/pendulum-3.1.0.tar.gz", hash = "sha256:66f96303560f41d097bee7d2dc98ffca716fbb3a832c4b3062034c2d45865015", size = 85930, upload-time = "2025-04-19T14:30:01.675Z" } wheels = [ @@ -1049,16 +1049,16 @@ wheels = [ [[package]] name = "pydantic-settings" -version = "2.12.0" +version = "2.14.2" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "pydantic" }, { name = "python-dotenv" }, { name = "typing-inspection" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/43/4b/ac7e0aae12027748076d72a8764ff1c9d82ca75a7a52622e67ed3f765c54/pydantic_settings-2.12.0.tar.gz", hash = "sha256:005538ef951e3c2a68e1c08b292b5f2e71490def8589d4221b95dab00dafcfd0", size = 194184, upload-time = "2025-11-10T14:25:47.013Z" } +sdist = { url = "https://files.pythonhosted.org/packages/5c/b5/8f48e906c3e0205276e8bd8cb7512217a87b2685304d64be27cad5b3019f/pydantic_settings-2.14.2.tar.gz", hash = "sha256:c19dd64b19097f1de80184f0cc7b0272a13ae6e170cbf240a3e27e381ed14a5f", size = 237700, upload-time = "2026-06-19T13:44:56.324Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/c1/60/5d4751ba3f4a40a6891f24eec885f51afd78d208498268c734e256fb13c4/pydantic_settings-2.12.0-py3-none-any.whl", hash = "sha256:fddb9fd99a5b18da837b29710391e945b1e30c135477f484084ee513adb93809", size = 51880, upload-time = "2025-11-10T14:25:45.546Z" }, + { url = "https://files.pythonhosted.org/packages/77/c1/6e422f34e569cf8e18df68d1939c81c099d2b61e4f7d9621c8a77560799c/pydantic_settings-2.14.2-py3-none-any.whl", hash = "sha256:a20c97b37910b6550d5ea50fbcc2d4187defe58cd57070b73863d069419c9440", size = 61715, upload-time = "2026-06-19T13:44:55.02Z" }, ] [[package]] @@ -1617,15 +1617,15 @@ wheels = [ [[package]] name = "starlette" -version = "1.2.1" +version = "1.3.1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "anyio" }, { name = "typing-extensions", marker = "python_full_version < '3.13'" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/25/44/ec35f1b6e83094b997da438a02c8c9b0ade2b1e84cfc48bd4656780760a6/starlette-1.2.1.tar.gz", hash = "sha256:9b9b5ebb992e67d6093741e63c2f59e4f6fff986f81163c087867bd7b924b3f6", size = 2701854, upload-time = "2026-05-31T01:07:51.847Z" } +sdist = { url = "https://files.pythonhosted.org/packages/eb/e3/7c1dc7381d9f8ab7d854328ebfa884e62cb3f3d8549ddfd37c7814f42afa/starlette-1.3.1.tar.gz", hash = "sha256:05d0213193f2fbaae60e2ecb593b4add4262ad4e46536b54abe36f11a71724e0", size = 2703240, upload-time = "2026-06-12T09:23:11.602Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/1c/54/196d0c1db10af76baa4f64894448505d60d3cdf70ef92cbb35f46a4e4c71/starlette-1.2.1-py3-none-any.whl", hash = "sha256:4de0082d08c8f6764a85a54cf1120d6939507a19905c7768acad2a9f875d2b89", size = 73350, upload-time = "2026-05-31T01:07:50.09Z" }, + { url = "https://files.pythonhosted.org/packages/ec/bb/2799cc2ede3ed41131f8975621e7213dfc7ef4acbbaadfa440f32500c370/starlette-1.3.1-py3-none-any.whl", hash = "sha256:c7372aae11c3c3f26a42df7bd626cec2f47d03483d261d369516a615a53714c6", size = 73632, upload-time = "2026-06-12T09:23:10.017Z" }, ] [[package]] @@ -1839,8 +1839,8 @@ name = "whenever" version = "0.9.3" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "tzdata", marker = "python_full_version >= '3.13' and sys_platform == 'win32'" }, - { name = "tzlocal", marker = "python_full_version >= '3.13' and sys_platform != 'darwin' and sys_platform != 'linux'" }, + { name = "tzdata", marker = "sys_platform == 'win32'" }, + { name = "tzlocal", marker = "sys_platform != 'darwin' and sys_platform != 'linux'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/87/ae/dcbfee50237cedca9153cac045dff6d93b81886f44f82c86493856592d55/whenever-0.9.3.tar.gz", hash = "sha256:22e6f8366767ab3c8be6d9e21a27bc56be2e50f0f2c66d78e8ee86497b579f1a", size = 256933, upload-time = "2025-10-16T19:44:40.832Z" } wheels = [ diff --git a/uv.lock b/uv.lock index 9c8c27b39d..a7a3de362f 100644 --- a/uv.lock +++ b/uv.lock @@ -708,55 +708,52 @@ wheels = [ [[package]] name = "cryptography" -version = "48.0.0" +version = "50.0.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "cffi", marker = "platform_python_implementation != 'PyPy'" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/9f/a9/db8f313fdcd85d767d4973515e1db101f9c71f95fced83233de224673757/cryptography-48.0.0.tar.gz", hash = "sha256:5c3932f4436d1cccb036cb0eaef46e6e2db91035166f1ad6505c3c9d5a635920", size = 832984, upload-time = "2026-05-04T22:59:38.133Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/df/3d/01f6dd9190170a5a241e0e98c2d04be3664a9e6f5b9b872cde63aff1c3dd/cryptography-48.0.0-cp311-abi3-macosx_10_9_universal2.whl", hash = "sha256:0c558d2cdffd8f4bbb30fc7134c74d2ca9a476f830bb053074498fbc86f41ed6", size = 8001587, upload-time = "2026-05-04T22:57:36.803Z" }, - { url = "https://files.pythonhosted.org/packages/b2/6e/e90527eef33f309beb811cf7c982c3aeffcce8e3edb178baa4ca3ae4a6fa/cryptography-48.0.0-cp311-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:f5333311663ea94f75dd408665686aaf426563556bb5283554a3539177e03b8c", size = 4690433, upload-time = "2026-05-04T22:57:40.373Z" }, - { url = "https://files.pythonhosted.org/packages/90/04/673510ed51ddff56575f306cf1617d80411ee76831ccd3097599140efdfe/cryptography-48.0.0-cp311-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:7995ef305d7165c3f11ae07f2517e5a4f1d5c18da1376a0a9ed496336b69e5f3", size = 4710620, upload-time = "2026-05-04T22:57:42.935Z" }, - { url = "https://files.pythonhosted.org/packages/14/d5/e9c4ef932c8d800490c34d8bd589d64a31d5890e27ec9e9ad532be893294/cryptography-48.0.0-cp311-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:40ba1f85eaa6959837b1d51c9767e230e14612eea4ef110ee8854ada22da1bf5", size = 4696283, upload-time = "2026-05-04T22:57:45.294Z" }, - { url = "https://files.pythonhosted.org/packages/0c/29/174b9dfb60b12d59ecfc6cfa04bc88c21b42a54f01b8aae09bb6e51e4c7f/cryptography-48.0.0-cp311-abi3-manylinux_2_28_ppc64le.whl", hash = "sha256:369a6348999f94bbd53435c894377b20ab95f25a9065c283570e70150d8abc3c", size = 5296573, upload-time = "2026-05-04T22:57:47.933Z" }, - { url = "https://files.pythonhosted.org/packages/95/38/0d29a6fd7d0d1373f0c0c88a04ba20e359b257753ac497564cd660fc1d55/cryptography-48.0.0-cp311-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:a0e692c683f4df67815a2d258b324e66f4738bd7a96a218c826dce4f4bd05d8f", size = 4743677, upload-time = "2026-05-04T22:57:50.067Z" }, - { url = "https://files.pythonhosted.org/packages/30/be/eef653013d5c63b6a490529e0316f9ac14a37602965d4903efed1399f32b/cryptography-48.0.0-cp311-abi3-manylinux_2_31_armv7l.whl", hash = "sha256:18349bbc56f4743c8b12dc32e2bccb2cf83ee8b69a3bba74ef8ae857e26b3d25", size = 4330808, upload-time = "2026-05-04T22:57:52.301Z" }, - { url = "https://files.pythonhosted.org/packages/84/9e/500463e87abb7a0a0f9f256ec21123ecde0a7b5541a15e840ea54551fd81/cryptography-48.0.0-cp311-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:7e8eac43dfca5c4cccc6dad9a80504436fca53bb9bc3100a2386d730fbe6b602", size = 4695941, upload-time = "2026-05-04T22:57:54.603Z" }, - { url = "https://files.pythonhosted.org/packages/e3/dc/7303087450c2ec9e7fbb750e17c2abfbc658f23cbd0e54009509b7cc4091/cryptography-48.0.0-cp311-abi3-manylinux_2_34_ppc64le.whl", hash = "sha256:9ccdac7d40688ecb5a3b4a604b8a88c8002e3442d6c60aead1db2a89a041560c", size = 5252579, upload-time = "2026-05-04T22:57:57.207Z" }, - { url = "https://files.pythonhosted.org/packages/d0/c0/7101d3b7215edcdc90c45da544961fd8ed2d6448f77577460fa75a8443f7/cryptography-48.0.0-cp311-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:bd72e68b06bb1e96913f97dd4901119bc17f39d4586a5adf2d3e47bc2b9d58b5", size = 4743326, upload-time = "2026-05-04T22:57:59.535Z" }, - { url = "https://files.pythonhosted.org/packages/ac/d8/5b833bad13016f562ab9d063d68199a4bd121d18458e439515601d3357ec/cryptography-48.0.0-cp311-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:59baa2cb386c4f0b9905bd6eb4c2a79a69a128408fd31d32ca4d7102d4156321", size = 4826672, upload-time = "2026-05-04T22:58:01.996Z" }, - { url = "https://files.pythonhosted.org/packages/98/e1/7074eb8bf3c135558c73fc2bcf0f5633f912e6fb87e868a55c454080ef09/cryptography-48.0.0-cp311-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:9249e3cd978541d665967ac2cb2787fd6a62bddf1e75b3e347a594d7dacf4f74", size = 4972574, upload-time = "2026-05-04T22:58:03.968Z" }, - { url = "https://files.pythonhosted.org/packages/04/70/e5a1b41d325f797f39427aa44ef8baf0be500065ab6d8e10369d850d4a4f/cryptography-48.0.0-cp311-abi3-win32.whl", hash = "sha256:9c459db21422be75e2809370b829a87eb37f74cd785fc4aa9ea1e5f43b47cda4", size = 3294868, upload-time = "2026-05-04T22:58:06.467Z" }, - { url = "https://files.pythonhosted.org/packages/f4/ac/8ac51b4a5fc5932eb7ee5c517ba7dc8cd834f0048962b6b352f00f41ebf9/cryptography-48.0.0-cp311-abi3-win_amd64.whl", hash = "sha256:5b012212e08b8dd5edc78ef54da83dd9892fd9105323b3993eff6bea65dc21d7", size = 3817107, upload-time = "2026-05-04T22:58:08.845Z" }, - { url = "https://files.pythonhosted.org/packages/6b/84/70e3feea9feea87fd7cbe77efb2712ae1e3e6edf10749dc6e95f4e60e455/cryptography-48.0.0-cp314-cp314t-macosx_10_9_universal2.whl", hash = "sha256:3cb07a3ed6431663cd321ea8a000a1314c74211f823e4177fefa2255e057d1ec", size = 7986556, upload-time = "2026-05-04T22:58:11.172Z" }, - { url = "https://files.pythonhosted.org/packages/89/6e/18e07a618bb5442ba10cf4df16e99c071365528aa570dfcb8c02e25a303b/cryptography-48.0.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:8c7378637d7d88016fa6791c159f698b3d3eed28ebf844ac36b9dc04a14dae18", size = 4684776, upload-time = "2026-05-04T22:58:13.712Z" }, - { url = "https://files.pythonhosted.org/packages/be/6a/4ea3b4c6c6759794d5ee2103c304a5076dc4b19ae1f9fe47dba439e159e9/cryptography-48.0.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:cc90c0b39b2e3c65ef52c804b72e3c58f8a04ab2a1871272798e5f9572c17d20", size = 4698121, upload-time = "2026-05-04T22:58:16.448Z" }, - { url = "https://files.pythonhosted.org/packages/2f/59/6ff6ad6cae03bb887da2a5860b2c9805f8dac969ef01ce563336c49bd1d1/cryptography-48.0.0-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:76341972e1eff8b4bea859f09c0d3e64b96ce931b084f9b9b7db8ef364c30eff", size = 4690042, upload-time = "2026-05-04T22:58:18.544Z" }, - { url = "https://files.pythonhosted.org/packages/ca/b4/fc334ed8cfd705aca282fe4d8f5ae64a8e0f74932e9feecb344610cf6e4d/cryptography-48.0.0-cp314-cp314t-manylinux_2_28_ppc64le.whl", hash = "sha256:55b7718303bf06a5753dcdccf2f3945cf18ad7bffde41b61226e4db31ab89a9c", size = 5282526, upload-time = "2026-05-04T22:58:20.75Z" }, - { url = "https://files.pythonhosted.org/packages/11/08/9f8c5386cc4cd90d8255c7cdd0f5baf459a08502a09de30dc51f553d38dc/cryptography-48.0.0-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:a64697c641c7b1b2178e573cbc31c7c6684cd56883a478d75143dbb7118036db", size = 4733116, upload-time = "2026-05-04T22:58:23.627Z" }, - { url = "https://files.pythonhosted.org/packages/b8/77/99307d7574045699f8805aa500fa0fb83422d115b5400a064ddd306d7750/cryptography-48.0.0-cp314-cp314t-manylinux_2_31_armv7l.whl", hash = "sha256:561215ea3879cb1cbbf272867e2efda62476f240fb58c64de6b393ae19246741", size = 4316030, upload-time = "2026-05-04T22:58:25.581Z" }, - { url = "https://files.pythonhosted.org/packages/fd/36/a608b98337af3cb2aff4818e406649d30572b7031918b04c87d979495348/cryptography-48.0.0-cp314-cp314t-manylinux_2_34_aarch64.whl", hash = "sha256:ad64688338ed4bc1a6618076ba75fd7194a5f1797ac60b47afe926285adb3166", size = 4689640, upload-time = "2026-05-04T22:58:27.747Z" }, - { url = "https://files.pythonhosted.org/packages/dd/a6/825010a291b4438aecc1f568bc428189fc1175515223632477c07dc0a6df/cryptography-48.0.0-cp314-cp314t-manylinux_2_34_ppc64le.whl", hash = "sha256:906cbf0670286c6e0044156bc7d4af9cbb0ef6db9f73e52c3ec56ba6bdde5336", size = 5237657, upload-time = "2026-05-04T22:58:29.848Z" }, - { url = "https://files.pythonhosted.org/packages/b9/09/4e76a09b4caa29aad535ddc806f5d4c5d01885bd978bd984fbc6ca032cae/cryptography-48.0.0-cp314-cp314t-manylinux_2_34_x86_64.whl", hash = "sha256:ea8990436d914540a40ab24b6a77c0969695ed52f4a4874c5137ccf7045a7057", size = 4732362, upload-time = "2026-05-04T22:58:32.009Z" }, - { url = "https://files.pythonhosted.org/packages/18/78/444fa04a77d0cb95f417dda20d450e13c56ba8e5220fc892a1658f44f882/cryptography-48.0.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:c18684a7f0cc9a3cb60328f496b8e3372def7c5d2df39ac267878b05565aaaae", size = 4819580, upload-time = "2026-05-04T22:58:34.254Z" }, - { url = "https://files.pythonhosted.org/packages/38/85/ea67067c70a1fd4be2c63d35eeed82658023021affccc7b17705f8527dd2/cryptography-48.0.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:9be5aafa5736574f8f15f262adc81b2a9869e2cfe9014d52a44633905b40d52c", size = 4963283, upload-time = "2026-05-04T22:58:36.376Z" }, - { url = "https://files.pythonhosted.org/packages/75/54/cc6d0f3deac3e81c7f847e8a189a12b6cdd65059b43dad25d4316abd849a/cryptography-48.0.0-cp314-cp314t-win32.whl", hash = "sha256:c17dfe85494deaeddc5ce251aebd1d60bbe6afc8b62071bb0b469431a000124f", size = 3270954, upload-time = "2026-05-04T22:58:38.791Z" }, - { url = "https://files.pythonhosted.org/packages/49/67/cc947e288c0758a4e5473d1dcb743037ab7785541265a969240b8885441a/cryptography-48.0.0-cp314-cp314t-win_amd64.whl", hash = "sha256:27241b1dc9962e056062a8eef1991d02c3a24569c95975bd2322a8a52c6e5e12", size = 3797313, upload-time = "2026-05-04T22:58:40.746Z" }, - { url = "https://files.pythonhosted.org/packages/f2/63/61d4a4e1c6b6bab6ce1e213cd36a24c415d90e76d78c5eb8577c5541d2e8/cryptography-48.0.0-cp39-abi3-macosx_10_9_universal2.whl", hash = "sha256:58d00498e8933e4a194f3076aee1b4a97dfec1a6da444535755822fe5d8b0b86", size = 7983482, upload-time = "2026-05-04T22:58:43.769Z" }, - { url = "https://files.pythonhosted.org/packages/d5/ac/f5b5995b87770c693e2596559ffafe195b4033a57f14a82268a2842953f3/cryptography-48.0.0-cp39-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:614d0949f4790582d2cc25553abd09dd723025f0c0e7c67376a1d77196743d6e", size = 4683266, upload-time = "2026-05-04T22:58:46.064Z" }, - { url = "https://files.pythonhosted.org/packages/ec/c6/8b14f67e18338fbc4adb76f66c001f5c3610b3e2d1837f268f47a347dbbb/cryptography-48.0.0-cp39-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:7ce4bfae76319a532a2dc68f82cc32f5676ee792a983187dac07183690e5c66f", size = 4696228, upload-time = "2026-05-04T22:58:48.22Z" }, - { url = "https://files.pythonhosted.org/packages/ea/73/f808fbae9514bd91b47875b003f13e284c8c6bdfd904b7944e803937eec1/cryptography-48.0.0-cp39-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:2eb992bbd4661238c5a397594c83f5b4dc2bc5b848c365c8f991b6780efcc5c7", size = 4689097, upload-time = "2026-05-04T22:58:50.9Z" }, - { url = "https://files.pythonhosted.org/packages/93/01/d86632d7d28db8ae83221995752eeb6639ffb374c2d22955648cf8d52797/cryptography-48.0.0-cp39-abi3-manylinux_2_28_ppc64le.whl", hash = "sha256:22a5cb272895dce158b2cacdfdc3debd299019659f42947dbdac6f32d68fe832", size = 5283582, upload-time = "2026-05-04T22:58:53.017Z" }, - { url = "https://files.pythonhosted.org/packages/02/e1/50edc7a50334807cc4791fc4a0ce7468b4a1416d9138eab358bfc9a3d70b/cryptography-48.0.0-cp39-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:2b4d59804e8408e2fea7d1fbaf218e5ec984325221db76e6a241a9abd6cdd95c", size = 4730479, upload-time = "2026-05-04T22:58:55.611Z" }, - { url = "https://files.pythonhosted.org/packages/6f/af/99a582b1b1641ff5911ac559beb45097cf79efd4ead4657f578ef1af2d47/cryptography-48.0.0-cp39-abi3-manylinux_2_31_armv7l.whl", hash = "sha256:984a20b0f62a26f48a3396c72e4bc34c66e356d356bf370053066b3b6d54634a", size = 4326481, upload-time = "2026-05-04T22:58:57.607Z" }, - { url = "https://files.pythonhosted.org/packages/90/ee/89aa26a06ef0a7d7611788ffd571a7c50e368cc6a4d5eef8b4884e866edb/cryptography-48.0.0-cp39-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:5a5ed8fde7a1d09376ca0b40e68cd59c69fe23b1f9768bd5824f54681626032a", size = 4688713, upload-time = "2026-05-04T22:59:00.077Z" }, - { url = "https://files.pythonhosted.org/packages/70/ba/bcb1b0bb7a33d4c7c0c4d4c7874b4a62ae4f56113a5f4baefa362dfb1f0f/cryptography-48.0.0-cp39-abi3-manylinux_2_34_ppc64le.whl", hash = "sha256:8cd666227ef7af430aa5914a9910e0ddd703e75f039cef0825cd0da71b6b711a", size = 5238165, upload-time = "2026-05-04T22:59:02.317Z" }, - { url = "https://files.pythonhosted.org/packages/c9/70/ca4003b1ce5ca3dc3186ada51908c8a9b9ff7d5cab83cc0d43ee14ec144f/cryptography-48.0.0-cp39-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:9071196d81abc88b3516ac8cdfad32e2b66dd4a5393a8e68a961e9161ddc6239", size = 4729947, upload-time = "2026-05-04T22:59:05.255Z" }, - { url = "https://files.pythonhosted.org/packages/44/a0/4ec7cf774207905aef1a8d11c3750d5a1db805eb380ee4e16df317870128/cryptography-48.0.0-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:1e2d54c8be6152856a36f0882ab231e70f8ec7f14e93cf87db8a2ed056bf160c", size = 4822059, upload-time = "2026-05-04T22:59:07.802Z" }, - { url = "https://files.pythonhosted.org/packages/1e/75/a2e55f99c16fcac7b5d6c1eb19ad8e00799854d6be5ca845f9259eae1681/cryptography-48.0.0-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:a5da777e32ffed6f85a7b2b3f7c5cbc88c146bfcd0a1d7baf5fcc6c52ee35dd4", size = 4960575, upload-time = "2026-05-04T22:59:09.851Z" }, - { url = "https://files.pythonhosted.org/packages/b8/23/6e6f32143ab5d8b36ca848a502c4bcd477ae75b9e1677e3530d669062578/cryptography-48.0.0-cp39-abi3-win32.whl", hash = "sha256:77a2ccbbe917f6710e05ba9adaa25fb5075620bf3ea6fb751997875aff4ae4bd", size = 3279117, upload-time = "2026-05-04T22:59:12.019Z" }, - { url = "https://files.pythonhosted.org/packages/9d/9a/0fea98a70cf1749d41d738836f6349d97945f7c89433a259a6c2642eefeb/cryptography-48.0.0-cp39-abi3-win_amd64.whl", hash = "sha256:16cd65b9330583e4619939b3a3843eec1e6e789744bb01e7c7e2e62e33c239c8", size = 3792100, upload-time = "2026-05-04T22:59:14.884Z" }, +sdist = { url = "https://files.pythonhosted.org/packages/de/41/6cbdcf9142d00fe82836fbb51e503e58088575cf7a0fe1dbff6695bf0840/cryptography-50.0.0.tar.gz", hash = "sha256:eeac2acb5a20ed25e0ad6d1df9891a520b78b404266b6d11778f25d5d691a6c9", size = 880201, upload-time = "2026-07-31T14:25:10.11Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c5/5c/59086b4aac5e879d38ddbcf74e4be7ade89cebc3eb199a55da998c3bb46a/cryptography-50.0.0-cp311-abi3-macosx_11_0_arm64.whl", hash = "sha256:031e2d5dd4bb9caa3ca9c82e5a197fd8ae680232cee62603d1a813f3f07e3d03", size = 4001252, upload-time = "2026-07-31T14:23:33.331Z" }, + { url = "https://files.pythonhosted.org/packages/57/ef/8f2df13c7216bcad3e1c74e07f6e193d93e998e114f524a53877c9af27ad/cryptography-50.0.0-cp311-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:fd9192b7b70c573d7f214eb1ae35e00d359f6f5e4b27c7e21e30de1fc6204645", size = 4719554, upload-time = "2026-07-31T14:23:35.611Z" }, + { url = "https://files.pythonhosted.org/packages/d9/41/029086c34d91052fc3b88bcc8056f709a7c915c7a23b235a54eb800b1c97/cryptography-50.0.0-cp311-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:06a32a980526a6ab9a4b9bf8f7385800791e2bb960903cb6b530e4817509a3b7", size = 4702130, upload-time = "2026-07-31T14:23:37.635Z" }, + { url = "https://files.pythonhosted.org/packages/7d/ff/b6ce0954962e7f7b969f850a883744197bb3910bdfd7b6da162eab7d9f68/cryptography-50.0.0-cp311-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:a1b30560f2acc95aa8b2e06e716a13dbfc97314747b80d9707e307f77b40d6b3", size = 4725244, upload-time = "2026-07-31T14:23:39.471Z" }, + { url = "https://files.pythonhosted.org/packages/06/1e/63a1027cb7fec360a182208e1b7767d5aa1fe57be3d6aa856e69a321edc0/cryptography-50.0.0-cp311-abi3-manylinux_2_28_ppc64le.whl", hash = "sha256:8d89f3976b10b4ce31118de72329025f70d2c6ead14a8217c5514dd2c6d5a78f", size = 5342265, upload-time = "2026-07-31T14:23:41.286Z" }, + { url = "https://files.pythonhosted.org/packages/6b/72/a1116d683a6d7ece94590013882515de087edf9ef0e6292aae615a44df73/cryptography-50.0.0-cp311-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:b42a28c1844fd9de8f3f7d540e36b66f3a9c83fceac7170ebc7a6a19edd9dcae", size = 4734609, upload-time = "2026-07-31T14:23:43.139Z" }, + { url = "https://files.pythonhosted.org/packages/15/37/36a9c479bbe49acea2636c7fd3360d20f7b7e079c300352011c44850b181/cryptography-50.0.0-cp311-abi3-manylinux_2_31_armv7l.whl", hash = "sha256:900131fafd8aead39ac7dd3a7e833be754c17a95cfd91221636949fe4eb0aa8a", size = 4356517, upload-time = "2026-07-31T14:23:44.939Z" }, + { url = "https://files.pythonhosted.org/packages/32/98/8a151d64367204cbc63ec65d37502f1d9c53cf4bfc6ec3c532614dbec60d/cryptography-50.0.0-cp311-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:07949c449a1abcf60d1ee6e88956d89404c7df3c8258f46589e912988e551987", size = 4724529, upload-time = "2026-07-31T14:23:46.93Z" }, + { url = "https://files.pythonhosted.org/packages/22/f6/ec13b470172126464a86bf54d2294a46d29837fc51ba3e45d4047946fb5e/cryptography-50.0.0-cp311-abi3-manylinux_2_34_ppc64le.whl", hash = "sha256:f89831ef99dd7dd169ab06d63a831adb9e20a87aac6d380266bbda5823349169", size = 5299852, upload-time = "2026-07-31T14:23:48.851Z" }, + { url = "https://files.pythonhosted.org/packages/da/3a/f05e32c99d440c9bb891ea0e36c9091891e36be5a9a87ab2ee6ea20729f6/cryptography-50.0.0-cp311-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:82148ec5bddac30b51a5b3c1945075f896fa022cb93f8e4a01e9f6ee95292c5f", size = 4734462, upload-time = "2026-07-31T14:23:50.861Z" }, + { url = "https://files.pythonhosted.org/packages/ca/dc/bd72b26be8953f80625f63151efd38eee71c76ca6cf591c08ff34615a79e/cryptography-50.0.0-cp311-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:1489e263a8048bb8b6a8bac662eb2d402ea5d2b7b4699b72f385f1e2772db105", size = 4852708, upload-time = "2026-07-31T14:23:52.715Z" }, + { url = "https://files.pythonhosted.org/packages/27/20/c930314a2ab476d15dec966ec87e2e9637bb02b06106b12c0396c57bb603/cryptography-50.0.0-cp311-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:7cec5b856506da6defb290f30c9ee687d5f5e8cb0bd3f6459dde43b0b4fa40ef", size = 5004179, upload-time = "2026-07-31T14:23:54.887Z" }, + { url = "https://files.pythonhosted.org/packages/32/2e/c9db68a0c4bfa28e310707527c0ee3a2bd254104d2e02e68f368e197aa4c/cryptography-50.0.0-cp311-abi3-win_amd64.whl", hash = "sha256:bd1c592e4d5974f0d08d4888e432157adba757c66da0246918e43677fafa2d30", size = 3840395, upload-time = "2026-07-31T14:23:56.677Z" }, + { url = "https://files.pythonhosted.org/packages/c3/fb/951032a3bf22a5697c83183fb6294a4843772947a70e616c57b3ff5f522e/cryptography-50.0.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:49e7d93abdbd2990caced757e5fade25302f719c3c8fb6e6fff2dde98999fc41", size = 3989258, upload-time = "2026-07-31T14:23:58.881Z" }, + { url = "https://files.pythonhosted.org/packages/d4/67/91eb047e69c5e845f2f14b8a2e4a1aab0f283cb885531e9e22c8adb176bc/cryptography-50.0.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:19736989797678c6af1e55cd49055cdbcb55d8f6b5583ac5335f933aba9101dc", size = 4700648, upload-time = "2026-07-31T14:24:00.702Z" }, + { url = "https://files.pythonhosted.org/packages/30/82/85f0f7425c856b9f96459411eb12e74ef72df9caf6f8f15bf23a33ff131f/cryptography-50.0.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:80b63928fa35083b33966ce1efb70e5b9607181e49dcd1c22c8c005e319f667f", size = 4682442, upload-time = "2026-07-31T14:24:02.538Z" }, + { url = "https://files.pythonhosted.org/packages/1a/28/b555a365adff1cca2fbe7b9e487d68a40de6bc67ff2cb587473eb43de0e7/cryptography-50.0.0-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:d58c3db7cd6eed54e6c06744db55456b65ebd7492ddeae9c1e93cfca7aa857d3", size = 4707596, upload-time = "2026-07-31T14:24:04.394Z" }, + { url = "https://files.pythonhosted.org/packages/72/d8/f52538140cc719df62a01cf87d1c7142318d235817109d6f4054d7c352d6/cryptography-50.0.0-cp314-cp314t-manylinux_2_28_ppc64le.whl", hash = "sha256:df2a58a472f332225671c35b0a830208b86d004f82baa8530fa3782c85646533", size = 5314552, upload-time = "2026-07-31T14:24:06.31Z" }, + { url = "https://files.pythonhosted.org/packages/38/14/6120e5bd7c5aa022ad15424ba4d5c5269d0d9448ed4d55e492ea91e3c1c4/cryptography-50.0.0-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:11b74db56cdbe3cdee6e3f6982ecb70334fa10dce99ed58bf7894aaaa3b2a037", size = 4717113, upload-time = "2026-07-31T14:24:08.349Z" }, + { url = "https://files.pythonhosted.org/packages/fa/71/190bf38c3ee2e0f8efc9860ae100c9df4169742eef274b91e7aa1cb133b9/cryptography-50.0.0-cp314-cp314t-manylinux_2_31_armv7l.whl", hash = "sha256:f59e38625469987d7ef6d495323c55e7db6c212eaf6112267e0d3b565a2e9c9f", size = 4338580, upload-time = "2026-07-31T14:24:10.227Z" }, + { url = "https://files.pythonhosted.org/packages/3a/63/504ccfbbe61fd8aa983f7f146399cdf034c72c2fc55f5b2dfdcdcdb20c99/cryptography-50.0.0-cp314-cp314t-manylinux_2_34_aarch64.whl", hash = "sha256:ecfed7367f965a0328cfbdd70da860f15441f002f613185668c6e6ebf5a0ac11", size = 4707038, upload-time = "2026-07-31T14:24:12.169Z" }, + { url = "https://files.pythonhosted.org/packages/01/77/2cf79bbfc4d12ca106437a6e170d6aaa01a373e93093118aaaef0e801bd4/cryptography-50.0.0-cp314-cp314t-manylinux_2_34_ppc64le.whl", hash = "sha256:9aa87839c383bdbab6ef865787a1fb877af8dd03464c4400322726feaaadfc6d", size = 5273110, upload-time = "2026-07-31T14:24:14.38Z" }, + { url = "https://files.pythonhosted.org/packages/e5/45/8aae2972c520145377ea3559a605a899bebe227bf070b33cdb445929a9b9/cryptography-50.0.0-cp314-cp314t-manylinux_2_34_x86_64.whl", hash = "sha256:6ba6a53445bd3cfa809ef3ef5f1589aa6ba08784a1d962bf47d0940e871dab1c", size = 4716439, upload-time = "2026-07-31T14:24:16.415Z" }, + { url = "https://files.pythonhosted.org/packages/7b/20/4fe50b619a48c2525cc46e2dbc1ac490708d704be5d467bdaac6dc955682/cryptography-50.0.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:3f5735ffe4996d28b809371756219f5354864902a3b9e7c0b9ee87041209fc9c", size = 4837383, upload-time = "2026-07-31T14:24:18.553Z" }, + { url = "https://files.pythonhosted.org/packages/92/91/3a31366e183343d3703f8995c095f5734676bd6938118047e50fcf279eb4/cryptography-50.0.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:1b4a266766514614f8aa60416e71f2fc6e575d36e7bdc90f644fadb2f4b75b95", size = 4985772, upload-time = "2026-07-31T14:24:20.385Z" }, + { url = "https://files.pythonhosted.org/packages/74/9a/02ffe35b2853d121689871eb5dce862092562b3a1ed5cc98f1aaed441506/cryptography-50.0.0-cp314-cp314t-win_amd64.whl", hash = "sha256:12b9c6996425c76ea6c457ace4f3073e715b8c545add07cd1a8f3a4f90691269", size = 3816291, upload-time = "2026-07-31T14:24:22.125Z" }, + { url = "https://files.pythonhosted.org/packages/03/37/73d005be173aff344af30e9fd2a576575cb2391a7101d9cd3842e1fa8cce/cryptography-50.0.0-cp39-abi3-macosx_11_0_arm64.whl", hash = "sha256:ccdc4a71a4dabae05de219404f9f4abc38e3b58422177ff93d0da05967dafa07", size = 4036009, upload-time = "2026-07-31T14:24:24.122Z" }, + { url = "https://files.pythonhosted.org/packages/ff/c6/7a6202a534e32103a285b7834a120869557fe198d51d7cfe59754c8bda9c/cryptography-50.0.0-cp39-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:910e1d2668e7de9648f2bcee30e180db2a6b15c30f887d7c4c93ddf96e3992e3", size = 4745252, upload-time = "2026-07-31T14:24:26.118Z" }, + { url = "https://files.pythonhosted.org/packages/85/4f/0fa8c2f4428198f15d9ff8d63400e27afbf94ce833f6108da1eb3753f945/cryptography-50.0.0-cp39-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:a91296cb61e8df6f86d0c19cc4068228da256bf59bf86049fbd821084565327f", size = 4728939, upload-time = "2026-07-31T14:24:27.994Z" }, + { url = "https://files.pythonhosted.org/packages/d1/63/54dd723490ba2dc09b299682c10b38db38f159728bcaae8c591b8af2f22d/cryptography-50.0.0-cp39-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:e722f16708d854fe924790e051061f6704a472c3bac347b6fd88033ea8dd0dc5", size = 4748483, upload-time = "2026-07-31T14:24:30.254Z" }, + { url = "https://files.pythonhosted.org/packages/1d/dd/7c77d26285cc7f6991efce64a0f5b4f9383bfa5dd8c5033003eaf7db4cdb/cryptography-50.0.0-cp39-abi3-manylinux_2_28_ppc64le.whl", hash = "sha256:d764dcf130c428ef66786f866dd750f53182bc608813489915e9fc106bb0c82f", size = 5367599, upload-time = "2026-07-31T14:24:32.457Z" }, + { url = "https://files.pythonhosted.org/packages/46/c9/f60aed34c013f317f92817b6c171c2d22a78270fa41109bd4b08af26b194/cryptography-50.0.0-cp39-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:105110f43a471dbd0060b9c9516cb8a6a79233631a04cc2ba16f28323ac6e025", size = 4762647, upload-time = "2026-07-31T14:24:34.599Z" }, + { url = "https://files.pythonhosted.org/packages/be/f3/f9a0173b139372c3a48ed98154b45cc6b9de17c789d5ab552e621c293609/cryptography-50.0.0-cp39-abi3-manylinux_2_31_armv7l.whl", hash = "sha256:828743d939e9629bc267b8e2d08d8bb67cd4319c771a33d4b18b22dd8fb7440a", size = 4385197, upload-time = "2026-07-31T14:24:36.647Z" }, + { url = "https://files.pythonhosted.org/packages/d8/36/83bb81f6e569bc38e1e4a7bc80f29b46bb9601920bc455fc8e888f5d5742/cryptography-50.0.0-cp39-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:2a8183b489dc1f7f80f135780fadc1108f14b31b8a40411c7a5b17425f65f28b", size = 4748095, upload-time = "2026-07-31T14:24:39.493Z" }, + { url = "https://files.pythonhosted.org/packages/6b/16/d3008eff98c764979865834c3d386d4fd041b5f52e7f34fc29ac1a5eb515/cryptography-50.0.0-cp39-abi3-manylinux_2_34_ppc64le.whl", hash = "sha256:6e7d61120573a7f2cd94cc095f9e81f6967c61ccdf194285aa143ecec8e0b708", size = 5325948, upload-time = "2026-07-31T14:24:41.556Z" }, + { url = "https://files.pythonhosted.org/packages/9c/f8/d97f9603efda3888187bfdb893f26c41be4735c10631d05d284ee6b047c4/cryptography-50.0.0-cp39-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:37fdb0d0111f1e2ff07139dfb79f1b49531f8e213c46f1163dd7642979b58c47", size = 4762400, upload-time = "2026-07-31T14:24:43.636Z" }, + { url = "https://files.pythonhosted.org/packages/64/a2/4615c8f7d81a00b1d6e6afe19f694e1543582349fb5f4076f6cb5dc36485/cryptography-50.0.0-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:c87f62a3d3b9888ed0fdde100ec06aa61ca9cd44bad9057d1dff9a516b5f5bb9", size = 4878208, upload-time = "2026-07-31T14:24:45.522Z" }, + { url = "https://files.pythonhosted.org/packages/d2/1a/efcfb02f91407149a0dacffffab791f7e19bf6385f63b3666dc8b5e5c9c8/cryptography-50.0.0-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:65c2c3add92b45fd0709db8594536aea39c2a67af0e27ffcf049c498501140b7", size = 5037050, upload-time = "2026-07-31T14:24:47.697Z" }, + { url = "https://files.pythonhosted.org/packages/57/30/4a22984d4f1bdfb8c054f07a92bc176b97a3134cc1d6c4b3bffb1f3688b4/cryptography-50.0.0-cp39-abi3-win_amd64.whl", hash = "sha256:d24fead1d4d076e1bfb006dcec392074a3cd8d7b4fc8a595aa64073b2b7a96ba", size = 3874135, upload-time = "2026-07-31T14:24:50.085Z" }, ] [[package]] @@ -1559,9 +1556,9 @@ requires-dist = [ { name = "pyarrow", marker = "extra == 'object-transfer'", specifier = ">=14" }, { name = "pydantic", specifier = ">=2.12,<2.13" }, { name = "pydantic-settings", specifier = ">=2.10,<3.0" }, - { name = "pyjwt", specifier = "==2.12.1" }, + { name = "pyjwt", specifier = "==2.13.0" }, { name = "pytest", specifier = ">=9.0,<9.1" }, - { name = "python-multipart", specifier = "==0.0.27" }, + { name = "python-multipart", specifier = "==0.0.31" }, { name = "pyyaml", specifier = ">=6,<7" }, { name = "redis", extras = ["hiredis"], specifier = "==6.0.0" }, { name = "rich", specifier = ">=13,<14" }, @@ -2624,8 +2621,8 @@ name = "pendulum" version = "3.2.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "python-dateutil", marker = "python_full_version < '3.13'" }, - { name = "tzdata", marker = "python_full_version < '3.13'" }, + { name = "python-dateutil" }, + { name = "tzdata" }, ] sdist = { url = "https://files.pythonhosted.org/packages/cb/72/9a51afa0a822b09e286c4cb827ed7b00bc818dac7bd11a5f161e493a217d/pendulum-3.2.0.tar.gz", hash = "sha256:e80feda2d10fa3ff8b1526715f7d33dcb7e08494b3088f2c8a3ac92d4a4331ce", size = 86912, upload-time = "2026-01-30T11:22:24.093Z" } wheels = [ @@ -3234,16 +3231,16 @@ wheels = [ [[package]] name = "pydantic-settings" -version = "2.14.1" +version = "2.14.2" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "pydantic" }, { name = "python-dotenv" }, { name = "typing-inspection" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/07/60/1d1e59c9c90d54591469ada7d268251f71c24bdb765f1a8a832cee8c6653/pydantic_settings-2.14.1.tar.gz", hash = "sha256:e874d3bec7e787b0c9958277956ed9b4dd5de6a80e162188fdaff7c5e26fd5fa", size = 235551, upload-time = "2026-05-08T13:40:06.542Z" } +sdist = { url = "https://files.pythonhosted.org/packages/5c/b5/8f48e906c3e0205276e8bd8cb7512217a87b2685304d64be27cad5b3019f/pydantic_settings-2.14.2.tar.gz", hash = "sha256:c19dd64b19097f1de80184f0cc7b0272a13ae6e170cbf240a3e27e381ed14a5f", size = 237700, upload-time = "2026-06-19T13:44:56.324Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/ae/8d/f1af3832f5e6eb13ba94ee809e72b8ecb5eef226d27ee0bef7d963d943c7/pydantic_settings-2.14.1-py3-none-any.whl", hash = "sha256:6e3c7edfd8277687cdc598f56e5cff0e9bfff0910a3749deaa8d4401c3a2b9de", size = 60964, upload-time = "2026-05-08T13:40:04.958Z" }, + { url = "https://files.pythonhosted.org/packages/77/c1/6e422f34e569cf8e18df68d1939c81c099d2b61e4f7d9621c8a77560799c/pydantic_settings-2.14.2-py3-none-any.whl", hash = "sha256:a20c97b37910b6550d5ea50fbcc2d4187defe58cd57070b73863d069419c9440", size = 61715, upload-time = "2026-06-19T13:44:55.02Z" }, ] [[package]] @@ -3293,11 +3290,11 @@ wheels = [ [[package]] name = "pyjwt" -version = "2.12.1" +version = "2.13.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/c2/27/a3b6e5bf6ff856d2509292e95c8f57f0df7017cf5394921fc4e4ef40308a/pyjwt-2.12.1.tar.gz", hash = "sha256:c74a7a2adf861c04d002db713dd85f84beb242228e671280bf709d765b03672b", size = 102564, upload-time = "2026-03-13T19:27:37.25Z" } +sdist = { url = "https://files.pythonhosted.org/packages/3b/81/58d0ac84e1ef3a3843791d6954d94c0b33d526c75eeb1efbce9d0a4c4077/pyjwt-2.13.0.tar.gz", hash = "sha256:41571c89ca91598c79e8ef18a2d07367d4810fbbd6f637794879baf1b7703423", size = 107515, upload-time = "2026-05-21T19:54:36.618Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/e5/7a/8dd906bd22e79e47397a61742927f6747fe93242ef86645ee9092e610244/pyjwt-2.12.1-py3-none-any.whl", hash = "sha256:28ca37c070cad8ba8cd9790cd940535d40274d22f80ab87f3ac6a713e6e8454c", size = 29726, upload-time = "2026-03-13T19:27:35.677Z" }, + { url = "https://files.pythonhosted.org/packages/a3/5e/ecf12fdb62546d64385c158514e9b2b671f7832108ef2ecd2020ce0af2d1/pyjwt-2.13.0-py3-none-any.whl", hash = "sha256:66adcc2aff09b3f1bbd95fc1e1577df8ac8723c978552fd43304c8a290ac5728", size = 31274, upload-time = "2026-05-21T19:54:35.362Z" }, ] [[package]] @@ -3514,11 +3511,11 @@ wheels = [ [[package]] name = "python-multipart" -version = "0.0.27" +version = "0.0.31" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/69/9b/f23807317a113dc36e74e75eb265a02dd1a4d9082abc3c1064acd22997c4/python_multipart-0.0.27.tar.gz", hash = "sha256:9870a6a8c5a20a5bf4f07c017bd1489006ff8836cff097b6933355ee2b49b602", size = 44043, upload-time = "2026-04-27T10:51:26.649Z" } +sdist = { url = "https://files.pythonhosted.org/packages/64/7e/9b35ad8f3d9ca680f7c87a88f19612fdd8da9796c4d3b46e560ac79dcc4a/python_multipart-0.0.31.tar.gz", hash = "sha256:fc631183bb13e56db3158a4909908dfb2e23565286744e798241e63750e5d680", size = 46689, upload-time = "2026-06-04T08:27:49.014Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/99/78/4126abcbdbd3c559d43e0db7f7b9173fc6befe45d39a2856cc0b8ec2a5a6/python_multipart-0.0.27-py3-none-any.whl", hash = "sha256:6fccfad17a27334bd0193681b369f476eda3409f17381a2d65aa7df3f7275645", size = 29254, upload-time = "2026-04-27T10:51:24.997Z" }, + { url = "https://files.pythonhosted.org/packages/5e/1e/7f7f299527a5a8ad90acd5f2f78dfa6c8495c6301a3205106ea68a84de96/python_multipart-0.0.31-py3-none-any.whl", hash = "sha256:8408153d68a9773291fc1da39a8b85a50044bddbabd2dd72e9229776b7b15e28", size = 29996, upload-time = "2026-06-04T08:27:47.804Z" }, ] [[package]] @@ -4070,15 +4067,15 @@ wheels = [ [[package]] name = "starlette" -version = "1.2.1" +version = "1.3.1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "anyio" }, { name = "typing-extensions", marker = "python_full_version < '3.13'" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/25/44/ec35f1b6e83094b997da438a02c8c9b0ade2b1e84cfc48bd4656780760a6/starlette-1.2.1.tar.gz", hash = "sha256:9b9b5ebb992e67d6093741e63c2f59e4f6fff986f81163c087867bd7b924b3f6", size = 2701854, upload-time = "2026-05-31T01:07:51.847Z" } +sdist = { url = "https://files.pythonhosted.org/packages/eb/e3/7c1dc7381d9f8ab7d854328ebfa884e62cb3f3d8549ddfd37c7814f42afa/starlette-1.3.1.tar.gz", hash = "sha256:05d0213193f2fbaae60e2ecb593b4add4262ad4e46536b54abe36f11a71724e0", size = 2703240, upload-time = "2026-06-12T09:23:11.602Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/1c/54/196d0c1db10af76baa4f64894448505d60d3cdf70ef92cbb35f46a4e4c71/starlette-1.2.1-py3-none-any.whl", hash = "sha256:4de0082d08c8f6764a85a54cf1120d6939507a19905c7768acad2a9f875d2b89", size = 73350, upload-time = "2026-05-31T01:07:50.09Z" }, + { url = "https://files.pythonhosted.org/packages/ec/bb/2799cc2ede3ed41131f8975621e7213dfc7ef4acbbaadfa440f32500c370/starlette-1.3.1-py3-none-any.whl", hash = "sha256:c7372aae11c3c3f26a42df7bd626cec2f47d03483d261d369516a615a53714c6", size = 73632, upload-time = "2026-06-12T09:23:10.017Z" }, ] [[package]] From 738693e890a7dc77eb5a5ce851413627d5101636 Mon Sep 17 00:00:00 2001 From: Infrahub Date: Sat, 8 Aug 2026 06:29:13 +0000 Subject: [PATCH 19/26] feat(agents): add analyzing-ci-flakiness skill Adds a skill that mines GitHub Actions history for flaky tests using retry outcomes (failed attempt -> green re-run) and cross-PR recurrence as evidence. A bundled stdlib-only collector caches runs, failed-job logs, and a longitudinal per-test ledger under ~/ci-cache so successive invocations build trend data instead of re-querying the GitHub API. Co-Authored-By: Claude Fable 5 --- .../skills/analyzing-ci-flakiness/SKILL.md | 119 +++++++ .../analyzing-ci-flakiness/scripts/collect.py | 325 ++++++++++++++++++ 2 files changed, 444 insertions(+) create mode 100644 .agents/skills/analyzing-ci-flakiness/SKILL.md create mode 100755 .agents/skills/analyzing-ci-flakiness/scripts/collect.py diff --git a/.agents/skills/analyzing-ci-flakiness/SKILL.md b/.agents/skills/analyzing-ci-flakiness/SKILL.md new file mode 100644 index 0000000000..73e6bfe980 --- /dev/null +++ b/.agents/skills/analyzing-ci-flakiness/SKILL.md @@ -0,0 +1,119 @@ +--- +name: analyzing-ci-flakiness +description: >- + Analyzes recent CI failures on pull requests to identify flaky tests, using retry outcomes + (failed attempt → green re-run) and cross-PR recurrence as evidence, and maintains a local + longitudinal ledger so flakiness can be tracked over time. TRIGGER when: the user wants to find + flaky tests, correlate recent CI failures, check which tests fail across PRs or recover on + retry, or refresh the flakiness trend report. DO NOT TRIGGER when: babysitting a single PR's CI + until green → monitoring-pull-requests; diagnosing or fixing one specific failing test → the + bug-analysis skills. +argument-hint: "Optional base-branch glob(s) and window, e.g. `release-1.11 14` (default: all bases, last 7 days)" +compatibility: Requires the gh CLI authenticated against the repo. Python 3 (stdlib only). Writes a cache under ~/ci-cache. +metadata: + version: 0.1.0 + author: OpsMill +--- + +# CI Flakiness Analyzer + +## Introduction + +A test is *flaky* when its failure does not reproduce on the same code: the run was retried and +went green, or the same test fails on unrelated PRs. This skill mines both signals from GitHub +Actions history, downloads the failed job logs once into a local cache, and appends every +observation to a ledger (`~/ci-cache/-/ledger.jsonl`) so repeated invocations — +weekly, or ad hoc — accumulate trend data instead of starting from scratch. + +The mechanical part (fetching, caching, test-name extraction, known-signature classification) is +done by the bundled script. Your job is the judgment part: separating flakes from real +regressions, spotting new systemic signatures, and writing the report. + +## Step 1 — Parse arguments + +- Base-branch filter: any arguments that look like branch names or globs (`release-1.11`, + `release-*`, `stable`). Default: no filter (all PR bases), which is usually what "how flaky is + CI" means. Filter when the user names a branch. +- Window: a bare integer is a number of days (default 7). An ISO date means "since that date". + +## Step 2 — Collect + +Run the bundled collector (repo-root relative): + +```bash +python3 .agents/skills/analyzing-ci-flakiness/scripts/collect.py \ + [--base ...] [--days N | --since YYYY-MM-DD] [--repo owner/name] +``` + +It prints a JSON report to stdout and writes everything under +`~/ci-cache/-/windows/_/`: + +- `runs.jsonl` — every `pull_request` workflow run created in the window +- `failed_jobs_with_tests.json` — failed jobs of the interesting run-attempts, with extracted + failing tests, systemic-bucket tags, and a `recovered_same_run` flag +- `report-data.json` — headline numbers, ranked per-test table, and the ledger's weekly history +- `joblogs/.log` — raw logs (ANSI intact; strip with `sed 's/\x1b\[[0-9;]*m//g'`) + +Notes the script already accounts for — don't re-derive them: + +- The runs API's `pull_requests` field is empty for many runs; the script joins runs to PRs + through every PR head commit SHA as well. Don't trust the field alone. +- "Interesting attempts" = every earlier attempt of a retried run (that's what the retry fixed) + plus final attempts that failed. Runs cancelled on attempt 1 are concurrency noise and skipped. +- Logs already on disk are never re-downloaded; the ledger is deduplicated by (job, test). Old + logs expire on GitHub's side (~90 days) — an empty `joblogs/*.log` means expired, not passing. + +## Step 3 — Investigate what the script could not name + +For failed jobs with an empty `tests` list and no bucket tag, read the log yourself (grep for +`##[error]`, `FAILED`, `Error:`, `Timeout`). Two outcomes: + +- It matches a *new* systemic signature (infra failure that cascades over many tests). Add a + regex for it to `BUCKETS` in `collect.py` and to the table below, so future runs classify it. +- It's a genuine test failure the extraction regexes missed — note the test manually and + consider extending `extract_tests`. + +### Known systemic signatures (as of 2026-08 — keep in sync with `BUCKETS` in collect.py) + +| Bucket | Signature | Meaning | +|---|---|---| +| `stack-readiness` | `ServerNotResponsiveError … /api/schema/load` | Seeded testcontainers stack not ready; the whole pytest-playwright shard errors. One incident, not N flaky tests. | +| `vitest-mock-corruption` | `TypeError: vi.mocked(...).mockX is not a function` | vitest browser-mode module-mocking race; hits a different test file each time. | +| `prefect-setup-triggers-timeout` | `Setup triggers` task `ReadTimeout` | Prefect hang at session setup; downstream tests hit their own timeouts. | +| `neo4j-deadlock` | `Neo.TransientError.Transaction.DeadlockDetected` | Concurrent-write deadlock, usually integration suites under xdist. | +| `compose-boot-failure` | `docker compose … up --wait` non-zero exit | Stack never booted; job-level infra failure. | +| `sqlite-locked` | `sqlite3.OperationalError: database is locked` | Prefect's sqlite under contention. | + +## Step 4 — Judge: flake vs regression + +For each test in the ranked table, classify: + +- **Flaky (strong)** — fails on ≥2 unrelated PRs, or `recovered_on_retry > 0`. The more distinct + PRs, the stronger. +- **Flaky (weak)** — single occurrence with an infra-flavored error (locator timeout, transient + branch not found) and the PR later went green. List, but rank low. +- **Suspect regression, not a flake** — the same test fails on *every* attempt of the same + commit and the PR is still red, or the failures started only after a specific merge. Say so + explicitly; do not bury it in the flake list. Cross-check: does the test fail on any PR that + does not contain the suspect change? +- **Systemic bucket** — tests whose only failures carry a bucket tag are casualties, not causes. + Report the bucket (with incident count), not the individual tests. + +Different tests failing on successive attempts of the same run = two independent flakes, not a +regression. + +## Step 5 — Report + +Write `ANALYSIS.md` into the window directory, then give the user a summary. Lead with the +ranked flake candidates. Include: + +1. Headline numbers: PRs in scope, runs matched, retried runs, retried-and-recovered runs + (pure-flake evidence), hard failures. +2. Ranked flake candidates — test id, distinct PRs/runs, recovered-on-retry count, one-line + error cause. Group systemic buckets as single entries. +3. Suspected real regressions, clearly separated. +4. Trend — from `weekly_history` in `report-data.json`: which offenders are new this window, + which recur week over week, which disappeared (likely fixed). This section is the reason the + ledger exists; don't skip it once ≥2 windows of data exist. + +Do not propose fixes unless asked; the deliverable is the evidence-ranked candidate list. diff --git a/.agents/skills/analyzing-ci-flakiness/scripts/collect.py b/.agents/skills/analyzing-ci-flakiness/scripts/collect.py new file mode 100755 index 0000000000..29375cf92c --- /dev/null +++ b/.agents/skills/analyzing-ci-flakiness/scripts/collect.py @@ -0,0 +1,325 @@ +#!/usr/bin/env python3 +"""Collect CI failure data for flakiness analysis, incrementally, into a local cache. + +Fetches GitHub Actions runs for pull requests, matches them to PRs (via the +runs' ``pull_requests`` field *and* a head-SHA join, since the field is often +empty), identifies retried runs and failed attempts, downloads the failed job +logs, extracts failing test identifiers, classifies known systemic failure +signatures, and appends everything to a longitudinal ledger so successive +invocations build trend data. + +Only stdlib + the ``gh`` CLI (must be authenticated). Safe to re-run: the runs +listing is refreshed each time, but job logs already on disk are never +re-downloaded and the ledger is deduplicated. + +Usage: + collect.py [--repo OWNER/NAME] [--base GLOB ...] [--days N | --since YYYY-MM-DD] + [--cache DIR] + +Outputs (under /-/): + ledger.jsonl one record per (job, test) ever observed + windows/_/ this invocation's window + runs.jsonl all pull_request runs created in the window + failed_jobs_with_tests.json failed jobs of interesting attempts + tests + report-data.json ranked frequency table + headline numbers + joblogs/.log raw logs of failed jobs (ANSI codes intact) +""" + +from __future__ import annotations + +import argparse +import datetime as dt +import fnmatch +import json +import re +import subprocess +import sys +from collections import defaultdict +from pathlib import Path + +ANSI = re.compile(r"\x1b\[[0-9;]*m") + +# Known systemic failure signatures. When one matches a job log, the job is +# tagged with the bucket so per-test counts don't mistake an infra cascade for +# N independent flaky tests. Keep in sync with the table in SKILL.md. +BUCKETS: list[tuple[str, str]] = [ + ("stack-readiness", r"ServerNotResponsiveError: Unable to read from '[^']*/api/schema/load"), + ("vitest-mock-corruption", r"TypeError: (?:vi\.mocked\(\.\.\.\)|\w+)\.mock\w+ is not a function"), + ("prefect-setup-triggers-timeout", r"'Setup triggers'.*ReadTimeout|Task run encountered an exception ReadTimeout"), + ("neo4j-deadlock", r"Neo\.TransientError\.Transaction\.DeadlockDetected"), + ("compose-boot-failure", r"'docker', 'compose'.*'up', '--wait'.*non-zero exit status"), + ("sqlite-locked", r"sqlite3\.OperationalError\) database is locked"), +] + + +def gh(args: list[str], *, check: bool = True) -> str: + res = subprocess.run(["gh", *args], capture_output=True, text=True, errors="replace", check=False) # noqa: S603, S607 + if res.returncode != 0 and check: + raise RuntimeError(f"gh {' '.join(args[:3])}... failed: {res.stderr.strip()[:300]}") + return res.stdout + + +def gh_json_lines(args: list[str]) -> list[dict]: + out = gh(args) + return [json.loads(line) for line in out.splitlines() if line.strip()] + + +def list_prs(repo: str, since: dt.date, base_globs: list[str]) -> list[dict]: + # Look back further than the run window: a re-run in the window can belong + # to a PR whose updatedAt predates it. + pr_since = since - dt.timedelta(days=21) + prs = json.loads( + gh( + [ + "pr", + "list", + "--repo", + repo, + "--state", + "all", + "--limit", + "500", + "--search", + f"updated:>={pr_since.isoformat()}", + "--json", + "number,title,state,baseRefName,headRefName,updatedAt", + ] + ) + ) + if base_globs: + prs = [p for p in prs if any(fnmatch.fnmatch(p["baseRefName"], g) for g in base_globs)] + return prs + + +def pr_head_shas(repo: str, numbers: list[int]) -> dict[str, set[int]]: + sha2pr: dict[str, set[int]] = defaultdict(set) + for n in numbers: + out = gh(["api", f"repos/{repo}/pulls/{n}/commits?per_page=100", "--paginate", "--jq", ".[].sha"], check=False) + for sha in out.split(): + sha2pr[sha].add(n) + return sha2pr + + +def list_runs(repo: str, since: dt.date) -> list[dict]: + jq = ( + ".workflow_runs[] | {id, name, head_branch, head_sha, run_attempt, " + "conclusion, status, created_at, prs: [.pull_requests[] | " + "{number, base: .base.ref}]}" + ) + return gh_json_lines( + [ + "api", + f"repos/{repo}/actions/runs?event=pull_request&created=%3E%3D{since.isoformat()}&per_page=100", + "--paginate", + "--jq", + jq, + ] + ) + + +def failed_jobs_for_attempt(repo: str, run_id: int, attempt: int) -> list[dict]: + return gh_json_lines( + [ + "api", + f"repos/{repo}/actions/runs/{run_id}/attempts/{attempt}/jobs?per_page=100", + "--paginate", + "--jq", + '.jobs[] | select(.conclusion=="failure") | {id, name}', + ] + ) + + +def extract_tests(job_name: str, text: str) -> list[str]: + """Pull failing test identifiers out of a cleaned (ANSI-stripped) job log.""" + fails: set[str] = set() + # pytest — backend suites and the pytest-playwright e2e suite + for m in re.finditer(r"(?:FAILED|ERROR) ((?:backend/)?tests/\S+::\S+)", text): + fails.add(m.group(1).split(" - ")[0].rstrip(",")) + # legacy TS Playwright — numbered entries of the failure report + if "E2E-testing-playwright" in job_name: + for m in re.finditer(r"\d+\)\s+\[\w+\]\s+›\s+(tests/e2e/[^\n›]+)›([^\n]+)", text): + spec = m.group(1).strip().split(":")[0] + title = re.sub(r"\s+", " ", m.group(2)).strip()[:120] + fails.add(f"PW {spec} › {title}") + # vitest browser mode + if job_name == "frontend-tests": + for m in re.finditer(r"FAIL\s+\|?\s*\w*\s*\|?\s+(src/\S+\.test\.\w+)", text): + fails.add(f"VITEST {m.group(1)}") + return sorted(fails) + + +def classify(text: str) -> list[str]: + return [name for name, pat in BUCKETS if re.search(pat, text)] + + +def main() -> int: + ap = argparse.ArgumentParser(description=__doc__) + ap.add_argument("--repo", default="opsmill/infrahub") + ap.add_argument( + "--base", + action="append", + default=[], + help="base-branch glob(s) to keep, e.g. release-1.11 or 'release-*' (default: all)", + ) + ap.add_argument("--days", type=int, default=7) + ap.add_argument("--since", type=lambda s: dt.date.fromisoformat(s)) + ap.add_argument("--cache", type=Path, default=Path.home() / "ci-cache") + args = ap.parse_args() + + today = dt.datetime.now(tz=dt.timezone.utc).date() + since = args.since or today - dt.timedelta(days=args.days) + repo_dir = args.cache / args.repo.replace("/", "-") + win_dir = repo_dir / "windows" / f"{since.isoformat()}_{today.isoformat()}" + (win_dir / "joblogs").mkdir(parents=True, exist_ok=True) + ledger_path = repo_dir / "ledger.jsonl" + + prs = list_prs(args.repo, since, args.base) + pr_by_num = {p["number"]: p for p in prs} + print(f"[collect] {len(prs)} PRs in scope (bases: {args.base or 'all'})", file=sys.stderr) + + runs = list_runs(args.repo, since) + (win_dir / "runs.jsonl").write_text("".join(json.dumps(r) + "\n" for r in runs)) + print(f"[collect] {len(runs)} pull_request runs since {since}", file=sys.stderr) + + sha2pr = pr_head_shas(args.repo, list(pr_by_num)) + + matched = [] + for r in runs: + nums = {p["number"] for p in r["prs"] if p["number"] in pr_by_num} + nums |= sha2pr.get(r["head_sha"], set()) + nums = {n for n in nums if n in pr_by_num} + if nums: + r["pr_nums"] = sorted(nums) + matched.append(r) + + # Attempts worth reading: every earlier attempt of a retried run (those + # failures are what the retry "fixed"), plus the final attempt when it + # failed outright. Runs cancelled on attempt 1 are concurrency noise. + targets: list[tuple[dict, int]] = [ + (r, a) for r in matched for a in range(1, r["run_attempt"] + (r["conclusion"] == "failure")) + ] + + print(f"[collect] {len(matched)} runs matched to PRs, {len(targets)} run-attempts to inspect", file=sys.stderr) + + seen_ledger: set[str] = set() + if ledger_path.exists(): + for line in ledger_path.open(): + rec = json.loads(line) + seen_ledger.add(rec["dedup_key"]) + + jobs_out, ledger_new = [], [] + for r, attempt in targets: + try: + jobs = failed_jobs_for_attempt(args.repo, r["id"], attempt) + except RuntimeError as exc: + print(f"[collect] WARN jobs {r['id']}/{attempt}: {exc}", file=sys.stderr) + continue + for job in jobs: + log_path = win_dir / "joblogs" / f"{job['id']}.log" + if not log_path.exists() or log_path.stat().st_size == 0: + text = gh(["api", f"repos/{args.repo}/actions/jobs/{job['id']}/logs"], check=False) + log_path.write_text(text) # empty file = log expired/unavailable + text = ANSI.sub("", log_path.read_text(errors="replace")) + tests = extract_tests(job["name"], text) + buckets = classify(text) + recovered = attempt < r["run_attempt"] and r["conclusion"] == "success" + entry = { + "run": r["id"], + "attempt": attempt, + "final_attempt": r["run_attempt"], + "final_conclusion": r["conclusion"], + "recovered_same_run": recovered, + "run_created": r["created_at"], + "workflow": r["name"], + "prs": [{"number": n, "base": pr_by_num[n]["baseRefName"]} for n in r["pr_nums"]], + "job_id": job["id"], + "job": job["name"], + "tests": tests, + "buckets": buckets, + "log_ok": bool(text.strip()), + } + jobs_out.append(entry) + week = dt.datetime.fromisoformat(r["created_at"]).strftime("%G-W%V") + for test in tests or [""]: + key = f"{job['id']}:{test}" + if key in seen_ledger: + continue + seen_ledger.add(key) + ledger_new.append( + { + "dedup_key": key, + "fetched_at": today.isoformat(), + "week": week, + "repo": args.repo, + "test": test, + **{ + k: entry[k] + for k in ( + "run", + "attempt", + "final_conclusion", + "recovered_same_run", + "run_created", + "workflow", + "job_id", + "job", + "prs", + "buckets", + ) + }, + } + ) + + (win_dir / "failed_jobs_with_tests.json").write_text(json.dumps(jobs_out, indent=1)) + with ledger_path.open("a") as fh: + for rec in ledger_new: + fh.write(json.dumps(rec) + "\n") + + # Frequency table for this window + trend across ledger weeks + freq: dict[str, list[dict]] = defaultdict(list) + for e in jobs_out: + for t in e["tests"]: + freq[t].append(e) + table = [] + for test, entries in freq.items(): + table.append( + { + "test": test, + "distinct_runs": len({e["run"] for e in entries}), + "distinct_prs": len({p["number"] for e in entries for p in e["prs"]}), + "attempts": len(entries), + "recovered_on_retry": sum(e["recovered_same_run"] for e in entries), + "buckets": sorted({b for e in entries for b in e["buckets"]}), + "prs": sorted({p["number"] for e in entries for p in e["prs"]}), + } + ) + table.sort(key=lambda x: (-x["distinct_prs"], -x["distinct_runs"], x["test"])) + + weeks_hist: dict[str, dict[str, int]] = defaultdict(lambda: defaultdict(int)) + if ledger_path.exists(): + for line in ledger_path.open(): + rec = json.loads(line) + if rec["test"]: + weeks_hist[rec["test"]][rec["week"]] += 1 + + report = { + "window": {"since": since.isoformat(), "until": today.isoformat()}, + "base_filter": args.base or "all", + "prs_in_scope": len(prs), + "runs_matched": len(matched), + "runs_retried": sum(1 for r in matched if r["run_attempt"] > 1), + "runs_recovered_on_retry": sum(1 for r in matched if r["run_attempt"] > 1 and r["conclusion"] == "success"), + "runs_failed_final": sum(1 for r in matched if r["conclusion"] == "failure"), + "failed_jobs": len(jobs_out), + "ranked_tests": table, + "weekly_history": {t: dict(sorted(w.items())) for t, w in sorted(weeks_hist.items())}, + "new_ledger_records": len(ledger_new), + } + (win_dir / "report-data.json").write_text(json.dumps(report, indent=1)) + print(json.dumps(report, indent=1)) + print(f"[collect] window dir: {win_dir}", file=sys.stderr) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) From 9769864938857b625d45d70192b6f7d7374249d9 Mon Sep 17 00:00:00 2001 From: Infrahub Date: Sat, 8 Aug 2026 06:35:24 +0000 Subject: [PATCH 20/26] fix(agents): satisfy repo ruff/ty config in flakiness collector Split main() into focused helpers, use set.update over add-in-loop, escape the Playwright breadcrumb separator, and add explicit encodings. Verified against the release-1.11 window: identical results (180 runs matched, 38 failed jobs) and idempotent ledger. Co-Authored-By: Claude Fable 5 --- .../analyzing-ci-flakiness/scripts/collect.py | 274 +++++++++--------- 1 file changed, 144 insertions(+), 130 deletions(-) diff --git a/.agents/skills/analyzing-ci-flakiness/scripts/collect.py b/.agents/skills/analyzing-ci-flakiness/scripts/collect.py index 29375cf92c..f7f427465e 100755 --- a/.agents/skills/analyzing-ci-flakiness/scripts/collect.py +++ b/.agents/skills/analyzing-ci-flakiness/scripts/collect.py @@ -32,13 +32,16 @@ import fnmatch import json import re -import subprocess +import subprocess # noqa: S404 import sys from collections import defaultdict from pathlib import Path ANSI = re.compile(r"\x1b\[[0-9;]*m") +# Playwright's breadcrumb separator (U+203A) as it appears in job logs. +PW_SEP = "\u203a" + # Known systemic failure signatures. When one matches a job log, the job is # tagged with the bucket so per-test counts don't mistake an infra cascade for # N independent flaky tests. Keep in sync with the table in SKILL.md. @@ -51,6 +54,19 @@ ("sqlite-locked", r"sqlite3\.OperationalError\) database is locked"), ] +LEDGER_FIELDS = ( + "run", + "attempt", + "final_conclusion", + "recovered_same_run", + "run_created", + "workflow", + "job_id", + "job", + "prs", + "buckets", +) + def gh(args: list[str], *, check: bool = True) -> str: res = subprocess.run(["gh", *args], capture_output=True, text=True, errors="replace", check=False) # noqa: S603, S607 @@ -133,18 +149,21 @@ def extract_tests(job_name: str, text: str) -> list[str]: """Pull failing test identifiers out of a cleaned (ANSI-stripped) job log.""" fails: set[str] = set() # pytest — backend suites and the pytest-playwright e2e suite - for m in re.finditer(r"(?:FAILED|ERROR) ((?:backend/)?tests/\S+::\S+)", text): - fails.add(m.group(1).split(" - ")[0].rstrip(",")) + fails.update( + m.group(1).split(" - ")[0].rstrip(",") + for m in re.finditer(r"(?:FAILED|ERROR) ((?:backend/)?tests/\S+::\S+)", text) + ) # legacy TS Playwright — numbered entries of the failure report if "E2E-testing-playwright" in job_name: - for m in re.finditer(r"\d+\)\s+\[\w+\]\s+›\s+(tests/e2e/[^\n›]+)›([^\n]+)", text): + for m in re.finditer(rf"\d+\)\s+\[\w+\]\s+{PW_SEP}\s+(tests/e2e/[^\n{PW_SEP}]+){PW_SEP}([^\n]+)", text): spec = m.group(1).strip().split(":")[0] title = re.sub(r"\s+", " ", m.group(2)).strip()[:120] - fails.add(f"PW {spec} › {title}") + fails.add(f"PW {spec} {PW_SEP} {title}") # vitest browser mode if job_name == "frontend-tests": - for m in re.finditer(r"FAIL\s+\|?\s*\w*\s*\|?\s+(src/\S+\.test\.\w+)", text): - fails.add(f"VITEST {m.group(1)}") + fails.update( + f"VITEST {m.group(1)}" for m in re.finditer(r"FAIL\s+\|?\s*\w*\s*\|?\s+(src/\S+\.test\.\w+)", text) + ) return sorted(fails) @@ -152,6 +171,112 @@ def classify(text: str) -> list[str]: return [name for name, pat in BUCKETS if re.search(pat, text)] +def match_runs_to_prs(runs: list[dict], pr_by_num: dict[int, dict], sha2pr: dict[str, set[int]]) -> list[dict]: + matched = [] + for r in runs: + nums = {p["number"] for p in r["prs"] if p["number"] in pr_by_num} + nums |= {n for n in sha2pr.get(r["head_sha"], set()) if n in pr_by_num} + if nums: + r["pr_nums"] = sorted(nums) + matched.append(r) + return matched + + +def collect_failed_jobs( + repo: str, targets: list[tuple[dict, int]], pr_by_num: dict[int, dict], win_dir: Path +) -> list[dict]: + """Fetch failed jobs and their logs for each (run, attempt); build job entries.""" + jobs_out = [] + for r, attempt in targets: + try: + jobs = failed_jobs_for_attempt(repo, r["id"], attempt) + except RuntimeError as exc: + print(f"[collect] WARN jobs {r['id']}/{attempt}: {exc}", file=sys.stderr) + continue + for job in jobs: + log_path = win_dir / "joblogs" / f"{job['id']}.log" + if not log_path.exists() or log_path.stat().st_size == 0: + # empty file = log expired/unavailable on GitHub's side + log_path.write_text(gh(["api", f"repos/{repo}/actions/jobs/{job['id']}/logs"], check=False)) + text = ANSI.sub("", log_path.read_text(errors="replace")) + jobs_out.append( + { + "run": r["id"], + "attempt": attempt, + "final_attempt": r["run_attempt"], + "final_conclusion": r["conclusion"], + "recovered_same_run": attempt < r["run_attempt"] and r["conclusion"] == "success", + "run_created": r["created_at"], + "workflow": r["name"], + "prs": [{"number": n, "base": pr_by_num[n]["baseRefName"]} for n in r["pr_nums"]], + "job_id": job["id"], + "job": job["name"], + "tests": extract_tests(job["name"], text), + "buckets": classify(text), + "log_ok": bool(text.strip()), + } + ) + return jobs_out + + +def append_ledger(ledger_path: Path, jobs_out: list[dict], repo: str, today: dt.date) -> int: + seen: set[str] = set() + if ledger_path.exists(): + seen = {json.loads(line)["dedup_key"] for line in ledger_path.open(encoding="utf-8")} + added = 0 + with ledger_path.open("a", encoding="utf-8") as fh: + for entry in jobs_out: + week = dt.datetime.fromisoformat(entry["run_created"]).strftime("%G-W%V") + for test in entry["tests"] or [""]: + key = f"{entry['job_id']}:{test}" + if key in seen: + continue + seen.add(key) + record = { + "dedup_key": key, + "fetched_at": today.isoformat(), + "week": week, + "repo": repo, + "test": test, + **{k: entry[k] for k in LEDGER_FIELDS}, + } + fh.write(json.dumps(record) + "\n") + added += 1 + return added + + +def ranked_tests(jobs_out: list[dict]) -> list[dict]: + freq: dict[str, list[dict]] = defaultdict(list) + for e in jobs_out: + for t in e["tests"]: + freq[t].append(e) + table = [ + { + "test": test, + "distinct_runs": len({e["run"] for e in entries}), + "distinct_prs": len({p["number"] for e in entries for p in e["prs"]}), + "attempts": len(entries), + "recovered_on_retry": sum(e["recovered_same_run"] for e in entries), + "buckets": sorted({b for e in entries for b in e["buckets"]}), + "prs": sorted({p["number"] for e in entries for p in e["prs"]}), + } + for test, entries in freq.items() + ] + table.sort(key=lambda x: (-x["distinct_prs"], -x["distinct_runs"], x["test"])) + return table + + +def weekly_history(ledger_path: Path) -> dict[str, dict[str, int]]: + hist: dict[str, dict[str, int]] = {} + if ledger_path.exists(): + for line in ledger_path.open(encoding="utf-8"): + rec = json.loads(line) + if rec["test"]: + weeks = hist.setdefault(rec["test"], {}) + weeks[rec["week"]] = weeks.get(rec["week"], 0) + 1 + return {t: dict(sorted(w.items())) for t, w in sorted(hist.items())} + + def main() -> int: ap = argparse.ArgumentParser(description=__doc__) ap.add_argument("--repo", default="opsmill/infrahub") @@ -162,158 +287,47 @@ def main() -> int: help="base-branch glob(s) to keep, e.g. release-1.11 or 'release-*' (default: all)", ) ap.add_argument("--days", type=int, default=7) - ap.add_argument("--since", type=lambda s: dt.date.fromisoformat(s)) + ap.add_argument("--since", type=dt.date.fromisoformat) ap.add_argument("--cache", type=Path, default=Path.home() / "ci-cache") args = ap.parse_args() - today = dt.datetime.now(tz=dt.timezone.utc).date() + today = dt.datetime.now(tz=dt.UTC).date() since = args.since or today - dt.timedelta(days=args.days) repo_dir = args.cache / args.repo.replace("/", "-") win_dir = repo_dir / "windows" / f"{since.isoformat()}_{today.isoformat()}" (win_dir / "joblogs").mkdir(parents=True, exist_ok=True) - ledger_path = repo_dir / "ledger.jsonl" - prs = list_prs(args.repo, since, args.base) - pr_by_num = {p["number"]: p for p in prs} - print(f"[collect] {len(prs)} PRs in scope (bases: {args.base or 'all'})", file=sys.stderr) + pr_by_num = {p["number"]: p for p in list_prs(args.repo, since, args.base)} + print(f"[collect] {len(pr_by_num)} PRs in scope (bases: {args.base or 'all'})", file=sys.stderr) runs = list_runs(args.repo, since) (win_dir / "runs.jsonl").write_text("".join(json.dumps(r) + "\n" for r in runs)) print(f"[collect] {len(runs)} pull_request runs since {since}", file=sys.stderr) - sha2pr = pr_head_shas(args.repo, list(pr_by_num)) - - matched = [] - for r in runs: - nums = {p["number"] for p in r["prs"] if p["number"] in pr_by_num} - nums |= sha2pr.get(r["head_sha"], set()) - nums = {n for n in nums if n in pr_by_num} - if nums: - r["pr_nums"] = sorted(nums) - matched.append(r) + matched = match_runs_to_prs(runs, pr_by_num, pr_head_shas(args.repo, list(pr_by_num))) # Attempts worth reading: every earlier attempt of a retried run (those # failures are what the retry "fixed"), plus the final attempt when it # failed outright. Runs cancelled on attempt 1 are concurrency noise. - targets: list[tuple[dict, int]] = [ - (r, a) for r in matched for a in range(1, r["run_attempt"] + (r["conclusion"] == "failure")) - ] - + targets = [(r, a) for r in matched for a in range(1, r["run_attempt"] + (r["conclusion"] == "failure"))] print(f"[collect] {len(matched)} runs matched to PRs, {len(targets)} run-attempts to inspect", file=sys.stderr) - seen_ledger: set[str] = set() - if ledger_path.exists(): - for line in ledger_path.open(): - rec = json.loads(line) - seen_ledger.add(rec["dedup_key"]) - - jobs_out, ledger_new = [], [] - for r, attempt in targets: - try: - jobs = failed_jobs_for_attempt(args.repo, r["id"], attempt) - except RuntimeError as exc: - print(f"[collect] WARN jobs {r['id']}/{attempt}: {exc}", file=sys.stderr) - continue - for job in jobs: - log_path = win_dir / "joblogs" / f"{job['id']}.log" - if not log_path.exists() or log_path.stat().st_size == 0: - text = gh(["api", f"repos/{args.repo}/actions/jobs/{job['id']}/logs"], check=False) - log_path.write_text(text) # empty file = log expired/unavailable - text = ANSI.sub("", log_path.read_text(errors="replace")) - tests = extract_tests(job["name"], text) - buckets = classify(text) - recovered = attempt < r["run_attempt"] and r["conclusion"] == "success" - entry = { - "run": r["id"], - "attempt": attempt, - "final_attempt": r["run_attempt"], - "final_conclusion": r["conclusion"], - "recovered_same_run": recovered, - "run_created": r["created_at"], - "workflow": r["name"], - "prs": [{"number": n, "base": pr_by_num[n]["baseRefName"]} for n in r["pr_nums"]], - "job_id": job["id"], - "job": job["name"], - "tests": tests, - "buckets": buckets, - "log_ok": bool(text.strip()), - } - jobs_out.append(entry) - week = dt.datetime.fromisoformat(r["created_at"]).strftime("%G-W%V") - for test in tests or [""]: - key = f"{job['id']}:{test}" - if key in seen_ledger: - continue - seen_ledger.add(key) - ledger_new.append( - { - "dedup_key": key, - "fetched_at": today.isoformat(), - "week": week, - "repo": args.repo, - "test": test, - **{ - k: entry[k] - for k in ( - "run", - "attempt", - "final_conclusion", - "recovered_same_run", - "run_created", - "workflow", - "job_id", - "job", - "prs", - "buckets", - ) - }, - } - ) - + jobs_out = collect_failed_jobs(args.repo, targets, pr_by_num, win_dir) (win_dir / "failed_jobs_with_tests.json").write_text(json.dumps(jobs_out, indent=1)) - with ledger_path.open("a") as fh: - for rec in ledger_new: - fh.write(json.dumps(rec) + "\n") - - # Frequency table for this window + trend across ledger weeks - freq: dict[str, list[dict]] = defaultdict(list) - for e in jobs_out: - for t in e["tests"]: - freq[t].append(e) - table = [] - for test, entries in freq.items(): - table.append( - { - "test": test, - "distinct_runs": len({e["run"] for e in entries}), - "distinct_prs": len({p["number"] for e in entries for p in e["prs"]}), - "attempts": len(entries), - "recovered_on_retry": sum(e["recovered_same_run"] for e in entries), - "buckets": sorted({b for e in entries for b in e["buckets"]}), - "prs": sorted({p["number"] for e in entries for p in e["prs"]}), - } - ) - table.sort(key=lambda x: (-x["distinct_prs"], -x["distinct_runs"], x["test"])) - - weeks_hist: dict[str, dict[str, int]] = defaultdict(lambda: defaultdict(int)) - if ledger_path.exists(): - for line in ledger_path.open(): - rec = json.loads(line) - if rec["test"]: - weeks_hist[rec["test"]][rec["week"]] += 1 + new_records = append_ledger(repo_dir / "ledger.jsonl", jobs_out, args.repo, today) report = { "window": {"since": since.isoformat(), "until": today.isoformat()}, "base_filter": args.base or "all", - "prs_in_scope": len(prs), + "prs_in_scope": len(pr_by_num), "runs_matched": len(matched), "runs_retried": sum(1 for r in matched if r["run_attempt"] > 1), "runs_recovered_on_retry": sum(1 for r in matched if r["run_attempt"] > 1 and r["conclusion"] == "success"), "runs_failed_final": sum(1 for r in matched if r["conclusion"] == "failure"), "failed_jobs": len(jobs_out), - "ranked_tests": table, - "weekly_history": {t: dict(sorted(w.items())) for t, w in sorted(weeks_hist.items())}, - "new_ledger_records": len(ledger_new), + "ranked_tests": ranked_tests(jobs_out), + "weekly_history": weekly_history(repo_dir / "ledger.jsonl"), + "new_ledger_records": new_records, } (win_dir / "report-data.json").write_text(json.dumps(report, indent=1)) print(json.dumps(report, indent=1)) From 540b0428253bbaa73c1991e56b3e6641cfb0c696 Mon Sep 17 00:00:00 2001 From: Infrahub Date: Wed, 19 Aug 2026 08:11:16 +0000 Subject: [PATCH 21/26] feat(agents): harden flakiness collector from second-window dogfooding - Split date-range queries: the Actions runs API silently caps listings at 1000 results; the first pass over a 1321-run week dropped the oldest 321 runs. - Extract Playwright tests from hyphenated project names (e.g. [docs-regression-check]). - New systemic buckets: runner-oom (exit 137), docker network pool exhaustion, actions-download 429, and green-pytest-exit-1 (session-teardown abort after all tests pass). Co-Authored-By: Claude Fable 5 --- .../skills/analyzing-ci-flakiness/SKILL.md | 4 ++ .../analyzing-ci-flakiness/scripts/collect.py | 41 +++++++++++++------ 2 files changed, 33 insertions(+), 12 deletions(-) diff --git a/.agents/skills/analyzing-ci-flakiness/SKILL.md b/.agents/skills/analyzing-ci-flakiness/SKILL.md index 73e6bfe980..5a24677a3f 100644 --- a/.agents/skills/analyzing-ci-flakiness/SKILL.md +++ b/.agents/skills/analyzing-ci-flakiness/SKILL.md @@ -83,6 +83,10 @@ For failed jobs with an empty `tests` list and no bucket tag, read the log yours | `neo4j-deadlock` | `Neo.TransientError.Transaction.DeadlockDetected` | Concurrent-write deadlock, usually integration suites under xdist. | | `compose-boot-failure` | `docker compose … up --wait` non-zero exit | Stack never booted; job-level infra failure. | | `sqlite-locked` | `sqlite3.OperationalError: database is locked` | Prefect's sqlite under contention. | +| `runner-oom` | `Process completed with exit code 137` | Runner OOM/SIGKILL; the mass test failures in the same job are casualties, not flakes. | +| `docker-network-pool-exhausted` | `all predefined address pools have been fully subnetted` | Leaked compose networks exhausted the docker address pools on a self-hosted runner. | +| `actions-download-429` | `Failed to download action … 429` | GitHub rate-limited its own action download; pure platform flake. | +| `pytest-green-exit-1` | green pytest summary directly followed by exit 1 | Session-teardown/plugin abort after all tests passed (e.g. testcontainers result reporting). | ## Step 4 — Judge: flake vs regression diff --git a/.agents/skills/analyzing-ci-flakiness/scripts/collect.py b/.agents/skills/analyzing-ci-flakiness/scripts/collect.py index f7f427465e..19df9f5e71 100755 --- a/.agents/skills/analyzing-ci-flakiness/scripts/collect.py +++ b/.agents/skills/analyzing-ci-flakiness/scripts/collect.py @@ -39,6 +39,9 @@ ANSI = re.compile(r"\x1b\[[0-9;]*m") +# The Actions list-runs API silently returns at most this many results per query. +API_RESULT_CAP = 1000 + # Playwright's breadcrumb separator (U+203A) as it appears in job logs. PW_SEP = "\u203a" @@ -52,6 +55,15 @@ ("neo4j-deadlock", r"Neo\.TransientError\.Transaction\.DeadlockDetected"), ("compose-boot-failure", r"'docker', 'compose'.*'up', '--wait'.*non-zero exit status"), ("sqlite-locked", r"sqlite3\.OperationalError\) database is locked"), + ("runner-oom", r"Process completed with exit code 137|exit code: 137"), + ("docker-network-pool-exhausted", r"all predefined address pools have been fully subnetted"), + ("actions-download-429", r"Failed to download action .*429"), + # pytest summary is green (no "N failed") yet the process exits 1: a + # session-teardown/plugin abort, e.g. the testcontainers result reporting. + ( + "pytest-green-exit-1", + r"=+ \d+ passed(?:(?!\d+ failed)[^\n])*=+[^\n]*\n(?:[^\n]*\n){0,3}[^\n]*Process completed with exit code 1\.", + ), ] LEDGER_FIELDS = ( @@ -116,21 +128,26 @@ def pr_head_shas(repo: str, numbers: list[int]) -> dict[str, set[int]]: return sha2pr -def list_runs(repo: str, since: dt.date) -> list[dict]: +def _runs_query(repo: str, created: str) -> str: + return f"repos/{repo}/actions/runs?event=pull_request&created={created}&per_page=100" + + +def list_runs(repo: str, since: dt.date, until: dt.date) -> list[dict]: + """List runs in [since, until], splitting the date range to stay under the API's 1000-result cap.""" jq = ( ".workflow_runs[] | {id, name, head_branch, head_sha, run_attempt, " "conclusion, status, created_at, prs: [.pull_requests[] | " "{number, base: .base.ref}]}" ) - return gh_json_lines( - [ - "api", - f"repos/{repo}/actions/runs?event=pull_request&created=%3E%3D{since.isoformat()}&per_page=100", - "--paginate", - "--jq", - jq, - ] - ) + created = f"{since.isoformat()}..{until.isoformat()}" + total = int(gh(["api", _runs_query(repo, created).replace("per_page=100", "per_page=1"), "--jq", ".total_count"])) + if total > API_RESULT_CAP and since < until: + mid = since + (until - since) // 2 + print(f"[collect] {total} runs in {created} exceeds the API result cap; splitting", file=sys.stderr) + return list_runs(repo, since, mid) + list_runs(repo, mid + dt.timedelta(days=1), until) + if total > API_RESULT_CAP: + print(f"[collect] WARN {total} runs on {since} alone; the API returns only the newest results", file=sys.stderr) + return gh_json_lines(["api", _runs_query(repo, created), "--paginate", "--jq", jq]) def failed_jobs_for_attempt(repo: str, run_id: int, attempt: int) -> list[dict]: @@ -155,7 +172,7 @@ def extract_tests(job_name: str, text: str) -> list[str]: ) # legacy TS Playwright — numbered entries of the failure report if "E2E-testing-playwright" in job_name: - for m in re.finditer(rf"\d+\)\s+\[\w+\]\s+{PW_SEP}\s+(tests/e2e/[^\n{PW_SEP}]+){PW_SEP}([^\n]+)", text): + for m in re.finditer(rf"\d+\)\s+\[[\w-]+\]\s+{PW_SEP}\s+(tests/e2e/[^\n{PW_SEP}]+){PW_SEP}([^\n]+)", text): spec = m.group(1).strip().split(":")[0] title = re.sub(r"\s+", " ", m.group(2)).strip()[:120] fails.add(f"PW {spec} {PW_SEP} {title}") @@ -300,7 +317,7 @@ def main() -> int: pr_by_num = {p["number"]: p for p in list_prs(args.repo, since, args.base)} print(f"[collect] {len(pr_by_num)} PRs in scope (bases: {args.base or 'all'})", file=sys.stderr) - runs = list_runs(args.repo, since) + runs = list_runs(args.repo, since, today) (win_dir / "runs.jsonl").write_text("".join(json.dumps(r) + "\n" for r in runs)) print(f"[collect] {len(runs)} pull_request runs since {since}", file=sys.stderr) From afb26e5817a9c8be00445eae31d4426bfe215ec7 Mon Sep 17 00:00:00 2001 From: Infrahub Date: Wed, 19 Aug 2026 14:28:03 +0000 Subject: [PATCH 22/26] fix(agents): stop re-downloading expired CI job logs every collection An expired log wrote an empty file via gh(..., check=False), and the st_size == 0 branch re-fetched it on every collection, defeating the documented never-re-download guarantee. Downloads now distinguish HTTP 404/410 (log gone: durable empty sentinel, never re-fetched) from transient gh failures (no file written, warned, retried next run), so the download condition is a plain exists() check. Co-Authored-By: Claude Fable 5 --- .../analyzing-ci-flakiness/scripts/collect.py | 32 ++++++++++++++++--- 1 file changed, 27 insertions(+), 5 deletions(-) diff --git a/.agents/skills/analyzing-ci-flakiness/scripts/collect.py b/.agents/skills/analyzing-ci-flakiness/scripts/collect.py index 19df9f5e71..88bb573ae4 100755 --- a/.agents/skills/analyzing-ci-flakiness/scripts/collect.py +++ b/.agents/skills/analyzing-ci-flakiness/scripts/collect.py @@ -22,7 +22,7 @@ runs.jsonl all pull_request runs created in the window failed_jobs_with_tests.json failed jobs of interesting attempts + tests report-data.json ranked frequency table + headline numbers - joblogs/.log raw logs of failed jobs (ANSI codes intact) + joblogs/.log raw logs of failed jobs (ANSI intact; empty = expired) """ from __future__ import annotations @@ -92,6 +92,29 @@ def gh_json_lines(args: list[str]) -> list[dict]: return [json.loads(line) for line in out.splitlines() if line.strip()] +def fetch_job_log(repo: str, job_id: int, log_path: Path) -> None: + """Download one job log, distinguishing gone from transiently unavailable. + + On success the log is written to ``log_path``. On HTTP 404/410 (the log + expired or was deleted on GitHub's side) an empty file is written as a + durable sentinel so the job is never re-fetched. On any other failure + (rate limit, network) nothing is written, so the next collection retries. + """ + res = subprocess.run( # noqa: S603 + ["gh", "api", f"repos/{repo}/actions/jobs/{job_id}/logs"], # noqa: S607 + capture_output=True, + text=True, + errors="replace", + check=False, + ) + if res.returncode == 0: + log_path.write_text(res.stdout, encoding="utf-8") + elif "HTTP 404" in res.stderr or "HTTP 410" in res.stderr: + log_path.write_text("", encoding="utf-8") + else: + print(f"[collect] WARN log {job_id}: {res.stderr.strip()[:300]}", file=sys.stderr) + + def list_prs(repo: str, since: dt.date, base_globs: list[str]) -> list[dict]: # Look back further than the run window: a re-run in the window can belong # to a PR whose updatedAt predates it. @@ -212,10 +235,9 @@ def collect_failed_jobs( continue for job in jobs: log_path = win_dir / "joblogs" / f"{job['id']}.log" - if not log_path.exists() or log_path.stat().st_size == 0: - # empty file = log expired/unavailable on GitHub's side - log_path.write_text(gh(["api", f"repos/{repo}/actions/jobs/{job['id']}/logs"], check=False)) - text = ANSI.sub("", log_path.read_text(errors="replace")) + if not log_path.exists(): + fetch_job_log(repo, job["id"], log_path) + text = ANSI.sub("", log_path.read_text(errors="replace")) if log_path.exists() else "" jobs_out.append( { "run": r["id"], From ec9e367e00efb64f9433daaf465eb75214138da2 Mon Sep 17 00:00:00 2001 From: Infrahub Date: Wed, 19 Aug 2026 14:30:38 +0000 Subject: [PATCH 23/26] fix(agents): match both renderings of the sqlite locked signature The sqlite-locked bucket regex only matched the SQLAlchemy-wrapped form `(sqlite3.OperationalError) database is locked` while the SKILL.md table documented the raw `sqlite3.OperationalError:` form. In every cached incident log both renderings appear together (chained traceback), so the bucket did tag, but a raw-only occurrence is plausible. Match both via `[):]` and sync the SKILL.md table with the code. Co-Authored-By: Claude Fable 5 --- .agents/skills/analyzing-ci-flakiness/SKILL.md | 2 +- .agents/skills/analyzing-ci-flakiness/scripts/collect.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.agents/skills/analyzing-ci-flakiness/SKILL.md b/.agents/skills/analyzing-ci-flakiness/SKILL.md index 5a24677a3f..847695d274 100644 --- a/.agents/skills/analyzing-ci-flakiness/SKILL.md +++ b/.agents/skills/analyzing-ci-flakiness/SKILL.md @@ -82,7 +82,7 @@ For failed jobs with an empty `tests` list and no bucket tag, read the log yours | `prefect-setup-triggers-timeout` | `Setup triggers` task `ReadTimeout` | Prefect hang at session setup; downstream tests hit their own timeouts. | | `neo4j-deadlock` | `Neo.TransientError.Transaction.DeadlockDetected` | Concurrent-write deadlock, usually integration suites under xdist. | | `compose-boot-failure` | `docker compose … up --wait` non-zero exit | Stack never booted; job-level infra failure. | -| `sqlite-locked` | `sqlite3.OperationalError: database is locked` | Prefect's sqlite under contention. | +| `sqlite-locked` | `(sqlite3.OperationalError) database is locked` (also matches the raw `sqlite3.OperationalError:` form) | Prefect's sqlite under contention. | | `runner-oom` | `Process completed with exit code 137` | Runner OOM/SIGKILL; the mass test failures in the same job are casualties, not flakes. | | `docker-network-pool-exhausted` | `all predefined address pools have been fully subnetted` | Leaked compose networks exhausted the docker address pools on a self-hosted runner. | | `actions-download-429` | `Failed to download action … 429` | GitHub rate-limited its own action download; pure platform flake. | diff --git a/.agents/skills/analyzing-ci-flakiness/scripts/collect.py b/.agents/skills/analyzing-ci-flakiness/scripts/collect.py index 88bb573ae4..a03ba1ac23 100755 --- a/.agents/skills/analyzing-ci-flakiness/scripts/collect.py +++ b/.agents/skills/analyzing-ci-flakiness/scripts/collect.py @@ -54,7 +54,7 @@ ("prefect-setup-triggers-timeout", r"'Setup triggers'.*ReadTimeout|Task run encountered an exception ReadTimeout"), ("neo4j-deadlock", r"Neo\.TransientError\.Transaction\.DeadlockDetected"), ("compose-boot-failure", r"'docker', 'compose'.*'up', '--wait'.*non-zero exit status"), - ("sqlite-locked", r"sqlite3\.OperationalError\) database is locked"), + ("sqlite-locked", r"sqlite3\.OperationalError[):] database is locked"), ("runner-oom", r"Process completed with exit code 137|exit code: 137"), ("docker-network-pool-exhausted", r"all predefined address pools have been fully subnetted"), ("actions-download-429", r"Failed to download action .*429"), From 02c54e2fce6b084d6cd1634335871361cfc6bd3d Mon Sep 17 00:00:00 2001 From: Infrahub Date: Wed, 19 Aug 2026 14:33:31 +0000 Subject: [PATCH 24/26] feat(agents): add per-bucket incident counts to the flakiness report MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The BUCKETS comment and SKILL.md Step 4 promised that a systemic cascade is reported as one incident, but report-data.json only annotated per-test entries with bucket tags — the analyst had to derive incident counts by hand from failed_jobs_with_tests.json. The report now carries bucket_incidents (distinct jobs/runs/PRs per bucket), the BUCKETS comment describes the actual mechanism (tags feed the incident counts and the Step 4 judgment), and SKILL.md points at the new field. Co-Authored-By: Claude Fable 5 --- .../skills/analyzing-ci-flakiness/SKILL.md | 7 ++++-- .../analyzing-ci-flakiness/scripts/collect.py | 23 ++++++++++++++++--- 2 files changed, 25 insertions(+), 5 deletions(-) diff --git a/.agents/skills/analyzing-ci-flakiness/SKILL.md b/.agents/skills/analyzing-ci-flakiness/SKILL.md index 847695d274..ea3cac0393 100644 --- a/.agents/skills/analyzing-ci-flakiness/SKILL.md +++ b/.agents/skills/analyzing-ci-flakiness/SKILL.md @@ -51,7 +51,9 @@ It prints a JSON report to stdout and writes everything under - `runs.jsonl` — every `pull_request` workflow run created in the window - `failed_jobs_with_tests.json` — failed jobs of the interesting run-attempts, with extracted failing tests, systemic-bucket tags, and a `recovered_same_run` flag -- `report-data.json` — headline numbers, ranked per-test table, and the ledger's weekly history +- `report-data.json` — headline numbers, ranked per-test table, per-bucket incident counts + (`bucket_incidents`: distinct jobs/runs/PRs per systemic bucket), and the ledger's weekly + history - `joblogs/.log` — raw logs (ANSI intact; strip with `sed 's/\x1b\[[0-9;]*m//g'`) Notes the script already accounts for — don't re-derive them: @@ -101,7 +103,8 @@ For each test in the ranked table, classify: explicitly; do not bury it in the flake list. Cross-check: does the test fail on any PR that does not contain the suspect change? - **Systemic bucket** — tests whose only failures carry a bucket tag are casualties, not causes. - Report the bucket (with incident count), not the individual tests. + Report the bucket (with the incident count from `bucket_incidents` in `report-data.json`), not + the individual tests. Different tests failing on successive attempts of the same run = two independent flakes, not a regression. diff --git a/.agents/skills/analyzing-ci-flakiness/scripts/collect.py b/.agents/skills/analyzing-ci-flakiness/scripts/collect.py index a03ba1ac23..5fa1ef5e2e 100755 --- a/.agents/skills/analyzing-ci-flakiness/scripts/collect.py +++ b/.agents/skills/analyzing-ci-flakiness/scripts/collect.py @@ -21,7 +21,7 @@ windows/_/ this invocation's window runs.jsonl all pull_request runs created in the window failed_jobs_with_tests.json failed jobs of interesting attempts + tests - report-data.json ranked frequency table + headline numbers + report-data.json ranked frequency table, per-bucket incident counts + headline numbers joblogs/.log raw logs of failed jobs (ANSI intact; empty = expired) """ @@ -46,8 +46,11 @@ PW_SEP = "\u203a" # Known systemic failure signatures. When one matches a job log, the job is -# tagged with the bucket so per-test counts don't mistake an infra cascade for -# N independent flaky tests. Keep in sync with the table in SKILL.md. +# tagged with the bucket. The tags feed the report's per-bucket incident counts +# (``bucket_incidents``) and the judgment step (SKILL.md Step 4), which reports +# a bucketed cascade as one incident rather than N flaky tests; the per-test +# table still lists every test, annotated with its buckets, so casualties can +# be discounted. Keep in sync with the table in SKILL.md. BUCKETS: list[tuple[str, str]] = [ ("stack-readiness", r"ServerNotResponsiveError: Unable to read from '[^']*/api/schema/load"), ("vitest-mock-corruption", r"TypeError: (?:vi\.mocked\(\.\.\.\)|\w+)\.mock\w+ is not a function"), @@ -305,6 +308,19 @@ def ranked_tests(jobs_out: list[dict]) -> list[dict]: return table +def bucket_incidents(jobs_out: list[dict]) -> dict[str, dict[str, int]]: + """Count distinct jobs/runs/PRs per systemic bucket, so a cascade reads as one incident.""" + jobs: dict[str, set[int]] = defaultdict(set) + runs: dict[str, set[int]] = defaultdict(set) + prs: dict[str, set[int]] = defaultdict(set) + for e in jobs_out: + for b in e["buckets"]: + jobs[b].add(e["job_id"]) + runs[b].add(e["run"]) + prs[b].update(p["number"] for p in e["prs"]) + return {b: {"jobs": len(jobs[b]), "runs": len(runs[b]), "prs": len(prs[b])} for b in sorted(jobs)} + + def weekly_history(ledger_path: Path) -> dict[str, dict[str, int]]: hist: dict[str, dict[str, int]] = {} if ledger_path.exists(): @@ -365,6 +381,7 @@ def main() -> int: "runs_failed_final": sum(1 for r in matched if r["conclusion"] == "failure"), "failed_jobs": len(jobs_out), "ranked_tests": ranked_tests(jobs_out), + "bucket_incidents": bucket_incidents(jobs_out), "weekly_history": weekly_history(repo_dir / "ledger.jsonl"), "new_ledger_records": new_records, } From cd972208e9a0b9adf72ccc1ee2ea3561efdbdf0c Mon Sep 17 00:00:00 2001 From: Infrahub Date: Wed, 19 Aug 2026 15:01:41 +0000 Subject: [PATCH 25/26] fix(agents): treat an empty successful log response as retryable A zero-byte body on a successful logs API call would have written the same empty file the expired-log (404/410) sentinel uses, permanently suppressing retries for that job. A real job log is never empty, so leave the file unwritten and warn instead; the next collection retries. Co-Authored-By: Claude Fable 5 --- .agents/skills/analyzing-ci-flakiness/scripts/collect.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.agents/skills/analyzing-ci-flakiness/scripts/collect.py b/.agents/skills/analyzing-ci-flakiness/scripts/collect.py index 5fa1ef5e2e..47abca6808 100755 --- a/.agents/skills/analyzing-ci-flakiness/scripts/collect.py +++ b/.agents/skills/analyzing-ci-flakiness/scripts/collect.py @@ -101,7 +101,9 @@ def fetch_job_log(repo: str, job_id: int, log_path: Path) -> None: On success the log is written to ``log_path``. On HTTP 404/410 (the log expired or was deleted on GitHub's side) an empty file is written as a durable sentinel so the job is never re-fetched. On any other failure - (rate limit, network) nothing is written, so the next collection retries. + (rate limit, network) — including a successful call with an empty body, + which a real job log never has — nothing is written, so the next + collection retries. """ res = subprocess.run( # noqa: S603 ["gh", "api", f"repos/{repo}/actions/jobs/{job_id}/logs"], # noqa: S607 @@ -110,8 +112,10 @@ def fetch_job_log(repo: str, job_id: int, log_path: Path) -> None: errors="replace", check=False, ) - if res.returncode == 0: + if res.returncode == 0 and res.stdout: log_path.write_text(res.stdout, encoding="utf-8") + elif res.returncode == 0: + print(f"[collect] WARN log {job_id}: empty response, leaving unfetched for retry", file=sys.stderr) elif "HTTP 404" in res.stderr or "HTTP 410" in res.stderr: log_path.write_text("", encoding="utf-8") else: From ee3c6258d3a0822b8c5af386650dc0cb9ad50456 Mon Sep 17 00:00:00 2001 From: Infrahub Date: Thu, 20 Aug 2026 07:52:59 +0000 Subject: [PATCH 26/26] chore(agents): pre-approve the flakiness collector via allowed-tools Running the skill no longer triggers a Bash permission prompt for the bundled collect.py invocation. Co-Authored-By: Claude Fable 5 --- .agents/skills/analyzing-ci-flakiness/SKILL.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.agents/skills/analyzing-ci-flakiness/SKILL.md b/.agents/skills/analyzing-ci-flakiness/SKILL.md index ea3cac0393..6c504490af 100644 --- a/.agents/skills/analyzing-ci-flakiness/SKILL.md +++ b/.agents/skills/analyzing-ci-flakiness/SKILL.md @@ -9,6 +9,8 @@ description: >- until green → monitoring-pull-requests; diagnosing or fixing one specific failing test → the bug-analysis skills. argument-hint: "Optional base-branch glob(s) and window, e.g. `release-1.11 14` (default: all bases, last 7 days)" +allowed-tools: + - Bash(python3 .agents/skills/analyzing-ci-flakiness/scripts/collect.py:*) compatibility: Requires the gh CLI authenticated against the repo. Python 3 (stdlib only). Writes a cache under ~/ci-cache. metadata: version: 0.1.0