From 704b09fa7a80e75b80c16f2323b91134c0238b63 Mon Sep 17 00:00:00 2001 From: Egor Kraev Date: Fri, 7 Aug 2026 11:41:04 +0200 Subject: [PATCH 1/7] fix: ingest resolves a schema scope and qualifies non-default schemas duckdb_engine's get_table_names(schema=None) returns objects from every schema as bare names -- every other Tier-1 dialect restricts it to the connection's default -- so _build_one_model wrote an unqualified sql_table for a non-default-schema object and the generator emitted `FROM reports`, which fails table-not-found. The same schema-blindness made _get_columns_fallback union two same-named tables' columns together. Ingest now resolves an explicit schema scope: --schema a,b / --all-schemas (schemas / all_schemas on the Python, REST and MCP surfaces), else datasource.schema_name, else the connection default. Multi-schema is opt-in because it changes what sql_table holds; when one schema is scanned and others exist, the run prints which and exits 0. Two different strings, easy to conflate: * the discovery token is carried exactly as get_schema_names() yields it (catalog-qualified on DuckDB), because the qualified form is the safe one -- with a catalog ATTACHed, a bare `main` makes get_table_names and has_table reach into it and makes the column fallback return the cross-catalog union. is_default therefore compares tokens in full. * the emitted qualifier is the bare last segment; the connection's current catalog is already correct. Both INFORMATION_SCHEMA fallbacks now filter on table_catalog as well as table_schema. table_schema alone holds the bare name, so a qualified token matched nothing: silently column-less models from the column fallback, and -- on DuckDB, where the Inspector reports no PK even for a declared PRIMARY KEY, so the fallback is the path that runs -- every primary key dropped. Only non-default schemas are qualified, so widening the scan never rewrites models already on disk; a single explicitly-named schema is written verbatim, preserving --schema public -> public.orders. Re-ingest heals a MISSING qualifier (participating in the short-circuit and the save gate, like source_kind) but never rewrites one, and two schemas' same-named tables are never fused into one model -- including the case a schema comparison alone cannot see, where the persisted model is unqualified because it IS the default schema's table. Collisions resolve in one phase over final model names with a 4-key total order, so the mixed sanitize/cross-schema case is defined and the outcome never depends on inspector listing order. validate-models derives its schema set from the models being validated and keys the live map on the full . identity, with shorter aliases inserted only when unique -- an ambiguous alias is dropped so a lookup misses rather than resolving to another catalog's same-named table. That is a data-loss path: an unresolvable model is a WholeModelDelete that --force-clean acts on. One dotted-name splitter replaces three disagreeing parsers, so hand-written Snowflake db.schema.table and BigQuery project.dataset.table stop losing their catalog. No new model field and no migration -- the schema lives in sql_table, which is where the generator already reads it. Closes DEV-1758 Co-Authored-By: Claude Opus 5 (1M context) --- .claude/skills/slayer-models.md | 9 + .claude/skills/slayer-overview.md | 2 +- DECISIONS.md | 1 + docs/concepts/ingestion.md | 39 +- docs/concepts/models.md | 12 + docs/configuration/datasources.md | 18 + docs/reference/cli.md | 45 +- slayer/api/server.py | 21 +- slayer/cli.py | 64 +- slayer/engine/ingestion.py | 877 +++++++-- slayer/engine/introspect_utils.py | 244 ++- slayer/engine/schema_drift.py | 133 +- slayer/mcp/server.py | 74 +- slayer/storage/type_refinement.py | 22 +- tests/test_ingestion.py | 14 +- tests/test_ingestion_schema_qualification.py | 1810 ++++++++++++++++++ 16 files changed, 3132 insertions(+), 253 deletions(-) create mode 100644 tests/test_ingestion_schema_qualification.py diff --git a/.claude/skills/slayer-models.md b/.claude/skills/slayer-models.md index 2031438d..8172daeb 100644 --- a/.claude/skills/slayer-models.md +++ b/.claude/skills/slayer-models.md @@ -88,6 +88,15 @@ Model names cannot contain `__` (reserved for join-path aliases), but `sql_table` can. Ingestion sanitizes only the name: object `reports__patient__drug` → model `reports_patient_drug`, `sql_table` unchanged. +`sql_table` is emitted verbatim into the generated SQL, so anything outside the +connection's default schema **must** be schema-qualified (`analytics.orders`) +or the query fails with table-not-found. Ingestion qualifies exactly when the +object's schema differs from the default, or when the schema was named +explicitly — default-schema objects stay bare, so both forms coexist in one +datasource by design. The schema is everything before the final dot, so +`project.dataset.table` works. A missing qualifier is repaired by re-ingesting +that schema; an existing one is never rewritten. + ## Query-backed models `create_model_from_query(query, name, variables=None)` saves a query (or list of stages) as a query-backed model. It populates `model.source_queries`, optional `model.query_variables` defaults, and caches `model.columns` + `model.backing_query_sql` from a save-time dry-run (unresolved `{var}` placeholders default to `'0'`). diff --git a/.claude/skills/slayer-overview.md b/.claude/skills/slayer-overview.md index 353955ab..da6db4b4 100644 --- a/.claude/skills/slayer-overview.md +++ b/.claude/skills/slayer-overview.md @@ -12,7 +12,7 @@ SLayer is a lightweight, agent-first semantic layer. Instead of writing raw SQL, - **SQLGenerator** — takes an EnrichedQuery (not SlayerQuery) and converts it to SQL via sqlglot (dialect-aware: postgres, mysql, bigquery, etc.) - **SlayerSQLClient** — executes SQL via SQLAlchemy with retry logic and statement timeouts - **Storage** — YAML or SQLite backends for model and datasource configs -- **Ingestion** — auto-generates models from DB schema with rollup-style FK joins (denormalized LEFT JOINs). It can be triggered manually (`slayer ingest`, `ingest_datasource_models`, `POST /ingest`) or **on every server boot** via `slayer serve --ingest-on-startup` / `slayer mcp --ingest-on-startup` (also `SLAYER_INGEST_ON_STARTUP=1`, or `create_app/create_mcp_server(ingest_on_startup=True)` programmatically). It is idempotent and continues on per-datasource failures. +- **Ingestion** — auto-generates models from DB schema with rollup-style FK joins (denormalized LEFT JOINs). It can be triggered manually (`slayer ingest`, `ingest_datasource_models`, `POST /ingest`) or **on every server boot** via `slayer serve --ingest-on-startup` / `slayer mcp --ingest-on-startup` (also `SLAYER_INGEST_ON_STARTUP=1`, or `create_app/create_mcp_server(ingest_on_startup=True)` programmatically). It is idempotent and continues on per-datasource failures. One pass covers **one schema** — `--schema a,b` / `--all-schemas` (and the `schemas` / `all_schemas` equivalents on MCP, REST and the Python API) opt into more; precedence is explicit flag → `datasource.schema_name` → the connection default. Non-default-schema objects get a schema-qualified `sql_table`. - **Interfaces** — MCP server (stdio via `slayer mcp`, SSE via `slayer serve` at `/mcp/sse`), REST API (FastAPI on port 5143), Python SDK, and two read-only wire-protocol facades for BI tools: Arrow Flight SQL (`slayer flight-serve`, port 5144) and Postgres (`slayer pg-serve`, port 5145; the connection `database` selects the SLayer datasource) ## Key Models diff --git a/DECISIONS.md b/DECISIONS.md index c5b1a4f0..67f149bc 100644 --- a/DECISIONS.md +++ b/DECISIONS.md @@ -70,3 +70,4 @@ implementation detail. Include issue refs when known. - 2026-08-04 — Dialect-aware / complete escaping for Mode-A `{variable}` substitution (DEV-1727), hardening DEV-1625. `substitute_variables(..., escape="sql")` is now **dialect-aware** and **fail-closed**: it gained a required keyword-only `backslash_escapes` signal (`bool | None`, raises if `None` in sql mode) so a caller rendering raw SQL can never silently under-escape. On backslash-escaping dialects (MySQL/ClickHouse/Snowflake/Redshift/BigQuery/Databricks/Spark) it doubles the backslash before escaping the single quote; on standard dialects it keeps the `''` quote-doubling. The double quote is deliberately left untouched — inside a single-quoted literal `\"` is NOT a recognised escape on 6 of the 7 backslash dialects (only MySQL), so escaping it would corrupt the value. The regime is DERIVED from sqlglot's own tokenizer via `SqlDialect.backslash_escapes_strings` (= `"\\" in tokenizer.STRING_ESCAPES`, guarded + 14-dialect pinned) so our escaping can never drift from the parser that reads the substituted SQL. `escape="python"` (Mode-B) additionally encodes the full C0 control range (`\t`/`\n`/`\r` named, rest `\xNN`) so raw newlines/NUL no longer break `ast.parse`. Engine fail-closed: `_substitute_model_sql_surfaces` / `_render_probe_model` require a `dialect`, threaded from the resolved datasource — no bare bool to forget. Assumes MySQL's default `sql_mode` (backslash escapes on); `NO_BACKSLASH_ESCAPES` servers are a sqlglot-layer-wide limitation, documented not fixed. The SQLite backslash end-to-end gap stays a pinned strict-xfail (pre-existing, out of scope). Bound parameters rejected (don't fit substitute-into-raw-SQL). Nested/join/cross-model lineages remain DEV-1678. - 2026-08-04 — Declared list-valued `{variable}` coercion (DEV-1730 follow-up): a scalar supplied for a variable the model declares `list_valued` is wrapped into a one-element list before Mode-A substitution, so an importer-generated `col IN ({var})` renders `IN ('US')` rather than the unquoted `IN (US)`. The generic scalar rule (author writes the quotes, so `{var}` also works in numeric/fragment positions like `amount >= {floor}` and `{d}::TIMESTAMP`) is CORRECT and unchanged — it just presumes an author who can see the SQL position, which a machine-generated fixed template does not have; the caller cannot supply per-element quotes through parentheses the importer wrote. Silent-wrong-answer risk drove the fix over a raise: `region IN (US)` parses as a column reference, so it fails at the database with a confusing message, or resolves against a real column and returns wrong rows. Opt-in is a front-end-NEUTRAL flag: the Cube converter writes `list_valued: ref.kind == "string"` into each `meta.cube_variables` entry (arrow forms splice pre-quoted scalars and stay `False`), and the engine reads only that flag — never Cube's `kind` taxonomy — so a future list-shaped front-end opts in the same way. Coercion lives at the single Mode-A choke point `_substitute_model_sql_surfaces` (execution and the `_render_probe_model` type-probe both route through it, so it cannot be bypassed) via `coerce_declared_list_variables` / `list_valued_variable_names` in `slayer/core/query.py`. Scope is deliberately narrow: only `str`/`int`/`float`/`bool` are wrapped; `list`/`tuple` pass through (the **empty list still raises** — "no filter" belongs to an optional block or a sentinel default); `None`/`dict` are left for `_render_variable_value` to reject with its own naming error; hand-written models declare nothing and are untouched. Follow-on from the same review: `declares_variables(model)` (any non-empty `meta.cube_variables`) now also defeats the DEV-1625 zero-variable fast path, via the shared `_model_needs_substitution_pass` predicate used by both `_substitute_model_sql_surfaces` and `_render_probe_model`. This closes the fast-path hole for a GENERATED model whose pushdowns are all required (no `{? ?}` block to force the pass): such a model used to emit a bare `{var}` into the SQL on a zero-variable call instead of raising the documented missing-variable error. The hole stays open — deliberately — for hand-written models, which declare nothing and keep the raw-brace-literal protection (`'{1,2,3}'`). The `list_valued` flag is matched with `is True`, not truthiness, since `meta` is user-extensible and a stray `1` or the string `"false"` must not switch substitution semantics. The bag is also SELF-IDENTIFYING — an entry counts only with a string `member` (the shape every importer writes) — so a hand-written `meta` that reuses the `cube_variables` key is not mistaken for generated SQL and silently stripped of its brace-literal protection. - 2026-08-05 — Ingestion sees views, survives unmodellable names, and stops being silent (DEV-1741). **Views**: `list_ingestable_objects` replaces the bare `get_table_names()` at every introspection site, adding `get_view_names` + `get_materialized_view_names` behind a `NotImplementedError`/`Exception` guard (the base `Inspector` RAISES for matviews on unsupporting dialects rather than returning `[]`), de-duplicated first-classification-wins because some dialects return views from `get_table_names()`, in a deterministic tables→views→matviews order that the name-collision policy depends on. Ingested by default — dbt materializes staging models as views, so opt-in would have left the reported failure in place for a fresh install — with `--no-views` on both `slayer ingest` and `datasources create --ingest`. The drift side (`_live_schema_for_datasource`) and the MCP listing (`_fetch_tables`) take **no flag and are unconditional**: that map is only ever a lookup target (`validate_datasource` iterates the *persisted* models, `available_in_ds` derives from them), so views there cannot manufacture a model or a drift entry — what they fix is a pre-existing **data-loss** bug where a hand-authored model whose `sql_table` named a view resolved to `live_table=None` → `WholeModelDelete` → deleted by `validate-models --force-clean`. Gating that on `--no-views` would re-arm it for exactly the users who opted out. **Names**: model names can't contain `__` (six modules — generator/enrichment/column_expansion/column_dependency/schema_drift/osi — `split("__")` an alias back into a join path, so a model named `a__b` is read as the alias for `a→b` and yields a silently wrong query, not a crash), but `sql_table` can, so a dlt child table `reports__patient__drug` is modelled as `reports_patient_drug` with `sql_table` verbatim. Sanitizer is `re.sub(r"_{2,}", "_")`, NOT `replace("__","_")` — `str.replace` is non-overlapping, so `a___b`→`a__b` would still fail validation. Collisions reserve every unsanitized name first (a real `a_b` always beats a sanitized `a__b`, order-independently) and **skip** rather than suffix, since suffixes shift as the object set changes and would orphan models and churn drift. A per-object `try/except` backstops everything else (`.`/`:`/`/`/`\` names, bad column names, per-object introspection failures); the FK-collection loop and `_get_fk_relationships` are guarded too because they run BEFORE that isolation and would otherwise still kill the run. **Reporting**: skips travel in a new `skipped` list, deliberately NOT folded into `errors` — `slayer ingest` exits 1 on either (we declined a perfectly valid object; `--exclude` is the documented remedy) but `POST /ingest` keeps 422 for `errors` only, because a permanent 422 aimed at a machine that can't act on the hint buries a successful partial ingest behind an error status. An empty scan prints the available schemas and exits 1 (the reporter listed the missing exit code as part of the defect), gated on `objects` not `additions` so a healthy no-op re-ingest stays quiet; `datasources create --ingest` on an empty DB still exits 0, since creating the datasource is that command's job and it succeeded. `in_scope_table_names` switched from model names to `_bare_table_name(sql_table)` — it is compared against table names in `_scoped_models_for_validation`, so the old keying silently dropped from validation scope any model whose name differs from its table (every sanitized model, and already every dbt/OSI hidden model passing `model_name=`). **`source_kind`** (`table`/`view`/`materialized_view`/`None`=unknown) persists on `SlayerModel`, v7→v8 with a no-op migration (mandatory: `migrate()` raises `RuntimeError` on an unregistered step); `None` for pre-v8, hand-authored and sql/query-backed models is the honest value, not a guess. It is a deliberate **exception to the additive-merge contract** — refreshed, not preserved — because it describes the live object rather than user intent, and the transition it exists to capture (dbt `+materialized: table`) usually changes no columns at all; the refresh therefore has to be made in three places (the early return, the `model_copy(update=...)`, and the save gate in `_process_one_table`), since editing only the update dict computes a corrected model and throws it away. A `None` from a non-classifying path never erases a known value. Docs-only fix for the advertised `motley-slayer[duckdb]` extra, which does not exist: `duckdb`/`duckdb-engine` are unconditional core deps (the Postgres facade imports duckdb at top level on every CLI invocation), and adding them under `[tool.poetry.extras]` would *gate* rather than alias them, breaking bare `pip install motley-slayer` → `datasources create demo`. +- 2026-08-07 — Ingest resolves a schema scope, and non-default schemas are written into `sql_table` (DEV-1758). **The bug**: `duckdb_engine`'s `get_table_names(schema=None)` returns objects from *every* schema as bare names — Postgres/MySQL/MSSQL/Snowflake/BigQuery/ClickHouse/SQLite all restrict it to the connection's default — so `_build_one_model`'s `f"{schema}.{name}" if schema else name` wrote an unqualified `sql_table` for a non-default-schema object and the generator emitted `FROM reports`, which fails table-not-found. The same schema-blindness made `_get_columns_fallback` union two same-named tables' columns. Not literally a #283 regression: that `sql_table` line is byte-identical before and after, and `--schema` always qualified correctly; #283 changed *visibility* by ingesting views by default, and dbt materialises staging models as views, so a dlt+dbt DuckDB file went from a handful of models to dozens, most unqueryable. **Scope**: one pass covers ONE schema — explicit `--schema a,b` / `--all-schemas` (plus `schemas` / `all_schemas` on the Python, REST and MCP surfaces), else `datasource.schema_name`, else the connection default. Multi-schema is opt-in because it changes what `sql_table` holds; when exactly one schema is scanned and others exist, the result carries a hint naming them and the exit code is unchanged (a hint is not a failure). `schema_name` is a **fallback**, never a conflict with an explicit flag; the genuine conflicts (`schema`+`schemas`, `all_schemas`+either) are rejected by one shared helper called from every entry point, so the CLI's mutually-exclusive group is not the only thing holding the line. **Two different strings**: the *discovery token* is carried exactly as `get_schema_names()` yields it — catalog-qualified on DuckDB (`fda.main`), bare elsewhere — and the qualified form is the SAFE one. Measured: with a second catalog `ATTACH`ed, a bare `main` makes `get_table_names` and `has_table` reach into the attached catalog and makes the column fallback return the cross-catalog union, while `att_main.main` is exact. So `is_default` compares the token IN FULL (no last-segment comparison, which is what made `att_main.main` and `other.main` both read as the default), and the INFORMATION_SCHEMA fallbacks filter on `table_catalog` as well as `table_schema` — `table_schema` alone holds the bare name, so a qualified token matched nothing and produced a silently **column-less** model, and retrying bare would resurrect the union. This applies to the PK fallback too, which on DuckDB is the path that actually runs (its Inspector reports no PK even for a declared PRIMARY KEY) — filtering it wrong drops every primary key, and fan-out safety leans on `Column.primary_key`. The *emitted qualifier* is a different string: the bare last segment, since the connection's current catalog is already correct. **What gets qualified** (D-9): only non-default schemas, so widening the scan never rewrites models already on disk and one datasource legitimately mixes both forms; a schema named explicitly as a single value is written verbatim, preserving today's `--schema public` → `public.orders`. A multi-schema list is deliberately NOT verbatim — listing the default alongside another schema would re-qualify every existing model. `--all-schemas` means the **current catalog only**; attached catalogs are dropped loudly with the exact `--schema .` invocation that ingests them, never silently. **Merging**: re-ingest heals a MISSING qualifier (participating in the short-circuit and the save gate, like `source_kind` — a repair usually changes no columns, so a merge that only edits the update dict computes the fix and discards it) but never rewrites an existing one. Two schemas' same-named tables are never fused into one model: a schema mismatch skips, and — the case a schema comparison alone cannot see, because D-9 persists default-schema models *unqualified* — a bare persisted `sql_table` that names a real default-schema object also skips, rather than being repointed at another schema's table by the heal. **Collisions** are resolved in ONE phase over final model names with a 4-key total order (unsanitized beats sanitized, then default schema, then schema name, then object name), not as successive passes, so the mixed case (`s1.a__b` sanitizing onto a real `s2.a_b`) is defined and the outcome never depends on inspector listing order. Losers skip, never suffix. **Validation** derives its schema set from the models being validated rather than gaining a flag, and the live map is keyed on the full `.` identity with shorter aliases inserted only when unique — an ambiguous alias is dropped so a lookup misses instead of resolving to another catalog's same-named table. This is a data-loss path, not a nuisance: an unresolvable model is a `WholeModelDelete` that `validate-models --force-clean` acts on. One dotted-name splitter (`split_sql_table`, everything before the FINAL dot) replaces three disagreeing parsers, so hand-written Snowflake `db.schema.table` and BigQuery `project.dataset.table` stop losing their catalog. No new model field and no v9 migration — the schema lives in `sql_table`, which is where the generator already reads it. diff --git a/docs/concepts/ingestion.md b/docs/concepts/ingestion.md index 09cff708..57521965 100644 --- a/docs/concepts/ingestion.md +++ b/docs/concepts/ingestion.md @@ -68,12 +68,36 @@ Non-SQLite datasources (Postgres, MySQL, DuckDB, ClickHouse, SQL Server) skip th Already-persisted v7 SQLite models with the wrong `INT` type are **not** auto-repaired on `storage.get_model()` load (running a full table scan per column on every load would be too expensive). Re-ingest is the auto-heal path: `slayer ingest` or `slayer serve --ingest-on-startup`. The DEV-1361 DOUBLE → INT narrowing on legacy-dict migration is also gated on the probe on SQLite — it only fires when the probe positively certifies INT. +## Schema scope + +One ingest pass covers one schema unless told otherwise. The scope is, in +order of precedence: the schemas named on the call; the datasource's +`schema_name`; the connection's default schema. Naming several schemas, or +every schema, is opt-in on all four surfaces: + +| Surface | One schema | Several | Every schema | +|---|---|---|---| +| CLI | `--schema public` | `--schema public,analytics` | `--all-schemas` | +| Python | `schemas=["public"]` | `schemas=["public", "analytics"]` | `all_schemas=True` | +| MCP | `schema_name="public"` | `schemas="public,analytics"` | `all_schemas=True` | +| REST | `"schema_name": "public"` | `"schemas": ["public","analytics"]` | `"all_schemas": true` | + +Combining two of them is rejected (a `ValueError`, a 422, or an error string) +rather than silently preferring one. When exactly one schema is scanned and +others exist, the result carries a hint naming them. + +Objects outside the connection's default schema are written with a +schema-qualified `sql_table`; default-schema objects stay unqualified. See +[`slayer ingest`](../reference/cli.md#which-schemas-get-ingested) for the full +rules, including qualifier repair and the cross-schema guard. + ## Usage ### CLI ```bash slayer ingest --datasource my_postgres --schema public --storage ./slayer_data +slayer ingest --datasource my_postgres --all-schemas --storage ./slayer_data ``` ### Python @@ -86,13 +110,15 @@ async def main(): result = await ingest_datasource_idempotent( datasource=ds, storage=storage, - schema="public", + schemas=["public"], # or all_schemas=True include_tables=["orders", "customers"], # Optional filter exclude_tables=["migrations"], # Optional exclusion ) - # result.additions — what was added (new models / columns / joins) - # result.to_delete — pending validate_models drift entries - # result.errors — per-model failures (best-effort, doesn't abort) + # result.additions — what was added (new models / columns / joins) + # result.to_delete — pending validate_models drift entries + # result.errors — per-model failures (best-effort, doesn't abort) + # result.skipped — live objects we declined to model, with reasons + # result.schema_hint — set when other schemas were left out return result asyncio.run(main()) @@ -103,6 +129,7 @@ asyncio.run(main()) ``` create_datasource(name="mydb", type="postgres", ...) ingest_datasource_models(datasource_name="mydb", schema_name="public") +ingest_datasource_models(datasource_name="mydb", all_schemas=True) ``` ### REST API @@ -111,6 +138,10 @@ ingest_datasource_models(datasource_name="mydb", schema_name="public") curl -X POST http://localhost:5143/ingest \ -H "Content-Type: application/json" \ -d '{"datasource": "my_postgres", "schema_name": "public"}' + +curl -X POST http://localhost:5143/ingest \ + -H "Content-Type: application/json" \ + -d '{"datasource": "my_postgres", "all_schemas": true}' ``` ## Querying Rolled-Up Models diff --git a/docs/concepts/models.md b/docs/concepts/models.md index 422b2a3c..09a09e4f 100644 --- a/docs/concepts/models.md +++ b/docs/concepts/models.md @@ -63,6 +63,18 @@ generated SQL, but `sql_table` has no such restriction. Auto-ingestion uses this: an object named `reports__patient__drug` becomes a model named `reports_patient_drug` whose `sql_table` is still `reports__patient__drug`. +`sql_table` may be schema-qualified (`analytics.orders`), and for anything +outside the connection's default schema it has to be — the generated SQL uses +the value verbatim, so an unqualified name resolves through the search path +and a non-default-schema table is simply not found. Auto-ingestion writes the +qualifier whenever the object's schema differs from the connection's default, +or whenever the schema was named explicitly; default-schema objects stay +unqualified. Within one datasource the two forms therefore coexist, which is +intended: it keeps widening the ingest scope from rewriting models that +already exist. Snowflake `db.schema.table` and BigQuery +`project.dataset.table` are accepted too — the schema is everything before the +final dot. + ## Columns A column is the unit of structure on the model. The same column entry can serve as a group-by key in one query and as input to an aggregation in another — the role is decided per query, not declared up front. What the column *carries* is its identity (name), how to compute it from the source (`sql`), what data type to expect, and a handful of policy fields (which aggregations are allowed, whether it's a primary key, whether it's hidden). diff --git a/docs/configuration/datasources.md b/docs/configuration/datasources.md index 4a332c75..293ae784 100644 --- a/docs/configuration/datasources.md +++ b/docs/configuration/datasources.md @@ -170,6 +170,24 @@ Statement-level timeout is enforced via !!! note Both `username` and `user` field names are accepted. The `user` alias is automatically mapped to `username` for compatibility with common database tooling conventions. +### `schema_name` and ingestion + +`schema_name` is the default schema for both `slayer ingest` and `slayer +validate-models`, so the two always look at the same tables. It is a +*fallback*: an explicit `--schema` / `--all-schemas` on the command line wins, +and neither combination is an error. With `schema_name` unset, ingest uses the +connection's default schema. + +`slayer datasources create --schema X --ingest` persists `schema_name: X`. A +comma-separated list or `--all-schemas` persists nothing — there is no single +value to record, and writing the first one would silently narrow every later +bare `slayer ingest`. + +Ingesting more than one schema is opt-in because it changes what `sql_table` +holds: objects outside the connection's default schema are written +schema-qualified (`analytics.orders`), which is what makes them queryable. +See [`slayer ingest`](../reference/cli.md#slayer-ingest). + ## Ingesting at Startup To run idempotent auto-ingestion across every configured datasource each time `slayer serve` or `slayer mcp` boots, pass `--ingest-on-startup` (or set `SLAYER_INGEST_ON_STARTUP=1`). See [Ingesting at Startup](../concepts/ingestion.md#ingesting-at-startup) for the full contract. diff --git a/docs/reference/cli.md b/docs/reference/cli.md index ef77b9e8..961d90ec 100644 --- a/docs/reference/cli.md +++ b/docs/reference/cli.md @@ -83,6 +83,8 @@ Auto-generate models from a datasource. ```bash slayer ingest --datasource my_postgres slayer ingest --datasource my_postgres --schema public +slayer ingest --datasource my_postgres --schema public,analytics +slayer ingest --datasource my_postgres --all-schemas slayer ingest --datasource my_postgres --include orders,customers slayer ingest --datasource my_postgres --exclude migrations,django_session slayer ingest --datasource my_postgres --no-views @@ -91,12 +93,50 @@ slayer ingest --datasource my_postgres --no-views | Flag | Required | Description | |------|----------|-------------| | `--datasource` | Yes | Datasource name | -| `--schema` | No | Database schema to inspect | +| `--schema` | No | Comma-separated schemas to inspect | +| `--all-schemas` | No | Inspect every non-system schema in the current database. Mutually exclusive with `--schema` | | `--include` | No | Comma-separated tables to include | | `--exclude` | No | Comma-separated tables to exclude | | `--no-views` | No | Skip views and materialized views (ingested by default) | | `--storage` | No | Storage path | +#### Which schemas get ingested + +With neither flag, ingest covers exactly one schema, resolved in this order: + +1. `--schema` / `--all-schemas`, when given; +2. the datasource's persisted `schema_name` + ([datasource config](../configuration/datasources.md)); +3. the connection's default schema. + +If other schemas exist, ingest names them and exits 0 — a hint, not a failure: + +``` +Note: ingested schema 'main' only. Other schemas in this datasource: openfda_rest. +Re-run with --schema openfda_rest, or --all-schemas, to ingest them. +``` + +Objects outside the connection's default schema get a schema-qualified +`sql_table` (`openfda_rest.reports`), which is what makes them queryable. +Default-schema objects stay unqualified, so widening the scan never rewrites +models that already exist. A schema named explicitly is always written +verbatim, so `--schema public` keeps producing `public.orders`. + +`--all-schemas` covers the **current database only**. Schemas belonging to an +`ATTACH`ed DuckDB catalog are reported as skipped, naming the +`--schema .` invocation that would ingest them. + +A model whose `sql_table` is missing its schema qualifier is repaired on the +next ingest of that schema, and the repair is reported: + +``` +Updated: reports (sql_table: reports → openfda_rest.reports) +``` + +An existing qualifier is never rewritten, and two schemas' same-named tables +are never merged into one model — the second is skipped with a `cross-schema` +reason rather than silently repointing the first. + #### Views Views and materialized views are ingested alongside tables by default — dbt @@ -205,7 +245,8 @@ slayer datasources create demo --ingest # bundled Jaffle Shop demo | `--name` | No | Override the auto-derived name (default for the demo: `jaffle_shop`) | | `--description` | No | Human-readable description | | `--ingest` | No | Run auto-ingestion immediately after creating the datasource | -| `--schema` | No | (with `--ingest`) Schema to ingest from | +| `--schema` | No | (with `--ingest`) Comma-separated schemas to ingest from. A single schema is also persisted as the datasource's `schema_name`, so later bare `slayer ingest` runs use it | +| `--all-schemas` | No | (with `--ingest`) Ingest every non-system schema. Mutually exclusive with `--schema`; persists no `schema_name` | | `--include` | No | (with `--ingest`) Comma-separated tables to include | | `--exclude` | No | (with `--ingest`) Comma-separated tables to exclude | | `--no-views` | No | (with `--ingest`) Skip views and materialized views (ingested by default) | diff --git a/slayer/api/server.py b/slayer/api/server.py index d1e6c0c1..e5caa9a3 100644 --- a/slayer/api/server.py +++ b/slayer/api/server.py @@ -5,7 +5,7 @@ from typing import Any from fastapi import FastAPI, HTTPException -from pydantic import BaseModel, ConfigDict, Field +from pydantic import BaseModel, ConfigDict, Field, model_validator from slayer.mcp.server import create_mcp_server from slayer.core.errors import ( @@ -103,7 +103,24 @@ class IngestRequest(BaseModel): datasource: str include_tables: list[str] | None = None exclude_tables: list[str] | None = None + # Kept for backward compatibility; folded into ``schemas=[schema_name]``. schema_name: str | None = None + schemas: list[str] | None = None + all_schemas: bool = False + + @model_validator(mode="after") + def _one_way_to_say_it(self) -> "IngestRequest": + """Reject conflicting scope arguments rather than silently preferring + whichever the handler reads first. The rule lives here so it applies + to every caller of the endpoint, and mirrors the engine's.""" + from slayer.engine.ingestion import _resolve_scope_args + + _resolve_scope_args( + schema=self.schema_name, + schemas=self.schemas, + all_schemas=self.all_schemas, + ) + return self class ValidateModelsRequest(BaseModel): @@ -650,6 +667,8 @@ async def ingest(request: IngestRequest) -> dict[str, Any]: include_tables=request.include_tables, exclude_tables=request.exclude_tables, schema=request.schema_name, + schemas=request.schemas, + all_schemas=request.all_schemas, ) except SQLAlchemyError as exc: # OperationalError / DatabaseError both derive from SQLAlchemyError diff --git a/slayer/cli.py b/slayer/cli.py index d4b09463..ea1a89a5 100644 --- a/slayer/cli.py +++ b/slayer/cli.py @@ -261,13 +261,30 @@ def main(): # NOSONAR(S3776) — linear top-level CLI command dispatch (one eli examples: slayer ingest --datasource my_postgres slayer ingest --datasource my_postgres --schema public + slayer ingest --datasource my_postgres --schema public,analytics + slayer ingest --datasource my_postgres --all-schemas slayer ingest --datasource my_postgres --include orders,customers slayer ingest --datasource my_postgres --exclude migrations,django_session """, formatter_class=argparse.RawDescriptionHelpFormatter, ) ingest_parser.add_argument("--datasource", required=True, help="Name of the datasource to ingest from") - ingest_parser.add_argument("--schema", default=None, help="Database schema to introspect (e.g., public)") + ingest_scope = ingest_parser.add_mutually_exclusive_group() + ingest_scope.add_argument( + "--schema", + default=None, + help=( + "Comma-separated schemas to introspect (e.g. public or " + "public,analytics). Default: the connection's default schema" + ), + ) + ingest_scope.add_argument( + "--all-schemas", + dest="all_schemas", + action="store_true", + default=False, + help="Introspect every non-system schema in the current database", + ) ingest_parser.add_argument( "--include", default=None, @@ -520,8 +537,23 @@ def main(): # NOSONAR(S3776) — linear top-level CLI command dispatch (one eli action="store_true", help="Run auto-ingestion immediately after creating the datasource", ) - datasources_create_parser.add_argument( - "--schema", default=None, help="(with --ingest) Schema to ingest from" + datasources_create_scope = ( + datasources_create_parser.add_mutually_exclusive_group() + ) + datasources_create_scope.add_argument( + "--schema", + default=None, + help=( + "(with --ingest) Comma-separated schemas to ingest from. A single " + "schema is also persisted as the datasource's default" + ), + ) + datasources_create_scope.add_argument( + "--all-schemas", + dest="all_schemas", + action="store_true", + default=False, + help="(with --ingest) Ingest every non-system schema in the database", ) datasources_create_parser.add_argument( "--include", @@ -1391,7 +1423,8 @@ def _run_ingest(args): ingest_datasource_idempotent( datasource=ds, storage=storage, - schema=args.schema, + schemas=_parse_csv_arg(args.schema), + all_schemas=getattr(args, "all_schemas", False), include_tables=_parse_csv_arg(args.include), exclude_tables=_parse_csv_arg(args.exclude), include_views=getattr(args, "include_views", True), @@ -1425,6 +1458,10 @@ def _run_ingest(args): for addition in result.additions: _print_ingest_addition(addition) _print_ingest_drift_and_errors(result) + # Narrowing the default scan to one schema is a behaviour change, so say + # which schemas were left out. Advisory only — the exit code is unchanged. + if getattr(result, "schema_hint", None): + print(f"\n{result.schema_hint}") # A skip means we declined to ingest a perfectly valid object, so it fails # the command — `--exclude ` is the documented way to make it green. if result.errors or result.skipped: @@ -1965,12 +2002,21 @@ def _run_datasources_create(args, storage): sys.exit(1) name = args.name or derived_name + schemas = _parse_csv_arg(args.schema) + all_schemas = getattr(args, "all_schemas", False) + # ``schema_name`` is a single-schema default. A CSV list or --all-schemas + # has no single value to persist, and persisting the first would silently + # narrow every later bare `slayer ingest`. + persisted_schema = ( + schemas[0] if schemas and len(schemas) == 1 and not all_schemas else None + ) ds = DatasourceConfig.model_validate( { "name": name, "type": ds_type, "connection_string": args.connection_string, "description": args.description, + "schema_name": persisted_schema, } ) @@ -1993,13 +2039,17 @@ def _run_datasources_create(args, storage): from slayer.engine.ingestion import ingest_datasource - include = [t for t in (s.strip() for s in args.include.split(",")) if t] if args.include else None - exclude = [t for t in (s.strip() for s in args.exclude.split(",")) if t] if args.exclude else None + include = _parse_csv_arg(args.include) + exclude = _parse_csv_arg(args.exclude) try: + # The parsed list is passed explicitly rather than relying on the + # persisted ``schema_name``, so the two can never be read as a + # conflict — ``schema=`` is deliberately never passed on this path. models = ingest_datasource( datasource=ds, - schema=args.schema, + schemas=schemas, + all_schemas=all_schemas, include_tables=include, exclude_tables=exclude, include_views=getattr(args, "include_views", True), diff --git a/slayer/engine/ingestion.py b/slayer/engine/ingestion.py index 50309d11..c041a14a 100644 --- a/slayer/engine/ingestion.py +++ b/slayer/engine/ingestion.py @@ -10,6 +10,7 @@ import asyncio import logging import sys +import warnings from collections import defaultdict, deque from typing import TYPE_CHECKING, Any, TextIO @@ -32,6 +33,9 @@ _get_columns_fallback, _parse_info_schema_is_float, _safe_get_columns, + qualified_default_schema, + split_schema_token, + split_sql_table, ) from slayer.core.errors import AmbiguousModelError, EntityResolutionError from slayer.memories.models import MEMORY_CANONICAL_PREFIX as _MEMORY_PREFIX @@ -452,19 +456,33 @@ def _get_pk_constraint_fallback( table_name: str, schema: str | None, ) -> dict: - """Get PK constraint via INFORMATION_SCHEMA when Inspector.get_pk_constraint() fails.""" + """Get PK constraint via INFORMATION_SCHEMA when Inspector.get_pk_constraint() fails. + + On DuckDB this is the path that actually runs — its Inspector reports an + empty ``constrained_columns`` even for a declared PRIMARY KEY — so it has + to understand the same catalog-qualified schema token discovery uses. + ``table_schema`` alone holds the bare name, and filtering on it with a + qualified token silently matched nothing, dropping every primary key. + """ if schema: + catalog, bare_schema = split_schema_token(schema) + clauses = [ + "tc.table_name = :table_name", + "tc.constraint_type = 'PRIMARY KEY'", + "tc.table_schema = :schema", + ] + params = {"table_name": table_name, "schema": bare_schema} + if catalog is not None: + clauses.append("tc.table_catalog = :catalog") + params["catalog"] = catalog sql = ( "SELECT kcu.column_name " "FROM information_schema.table_constraints tc " "JOIN information_schema.key_column_usage kcu " " ON tc.constraint_name = kcu.constraint_name " " AND tc.table_schema = kcu.table_schema " - "WHERE tc.table_name = :table_name " - " AND tc.constraint_type = 'PRIMARY KEY' " - " AND tc.table_schema = :schema" + "WHERE " + " AND ".join(clauses) ) - params = {"table_name": table_name, "schema": schema} else: sql = ( "SELECT kcu.column_name " @@ -726,14 +744,11 @@ def _sqlite_probe_integer_columns( def _parse_qualified_sql_table(sql_table: str) -> tuple[str | None, str]: """Split ``"schema.table"`` into ``(schema, table)`` or ``(None, table)``. - Only splits on a single dot — table/schema names containing dots are - out of scope for the auto-ingest path (the dotted form would never have - survived ``Inspector.get_table_names`` either). + Delegates to :func:`split_sql_table` so a three-part + ``catalog.schema.table`` keeps its catalog instead of losing it to a + split on the first dot. """ - if "." in sql_table: - schema, _, table = sql_table.partition(".") - return schema or None, table - return None, sql_table + return split_sql_table(sql_table) def introspect_table_to_model( @@ -784,11 +799,23 @@ def introspect_table_to_model( # --------------------------------------------------------------------------- -class IngestableObject(BaseModel): - """One database object discovered by :func:`list_ingestable_objects`.""" +with warnings.catch_warnings(): + # ``schema`` shadows Pydantic v2's deprecated ``BaseModel.schema()``. + # The name is deliberate — it is the SQLAlchemy ``Inspector`` keyword this + # value is passed as — and the shadowed classmethod is never called here. + warnings.filterwarnings("ignore", message='Field name "schema"') - name: str - kind: ObjectKind + class IngestableObject(BaseModel): + """One database object discovered by :func:`list_ingestable_objects`.""" + + name: str + kind: ObjectKind + # The discovery token the object was found under — catalog-qualified + # on dialects that qualify (DuckDB), bare elsewhere, ``None`` when the + # dialect reports no default schema. This is the string handed back to + # the Inspector; the qualifier written into ``sql_table`` is a + # different string (see :func:`qualify_sql_table`). + schema: str | None = None class SkippedTable(BaseModel): @@ -813,6 +840,221 @@ class IngestionScanReport(BaseModel): # but they were all skipped / already in sync" — the CLI needs that # distinction to decide between the empty-schema hint and silence. objects: list[IngestableObject] = Field(default_factory=list) + # Set when exactly one schema was scanned and the datasource holds + # others: narrowing the default scan is a behaviour change, so it has to + # be visible. Never an error — a hint is not a failure. + schema_hint: str | None = None + # Object names living in the connection's default schema. The additive + # pass needs it to tell "this persisted unqualified model IS the default + # schema's table" from "a same-named table in another schema", which is + # the one case a schema comparison alone cannot decide. + default_schema_objects: list[str] = Field(default_factory=list) + + +# --------------------------------------------------------------------------- +# Schema scope +# --------------------------------------------------------------------------- + + +_SYSTEM_SCHEMA_NAMES = frozenset( + { + "information_schema", + "pg_catalog", + "pg_toast", + "performance_schema", + "mysql", + "sys", + "sys_temp", + } +) +_SYSTEM_SCHEMA_PREFIXES = ("pg_temp_", "pg_toast_temp_") +# DuckDB exposes its own metadata and scratch space as first-segment +# catalogs (``system.main``, ``temp.main``), so those are matched on the +# catalog rather than on the schema name. +_SYSTEM_CATALOGS = frozenset({"system", "temp"}) + + +def _is_system_schema(token: str) -> bool: + """Whether a discovery token names a system schema rather than user data.""" + catalog, schema = split_schema_token(token) + if catalog is not None and catalog.lower() in _SYSTEM_CATALOGS: + return True + bare = schema.lower() + return bare in _SYSTEM_SCHEMA_NAMES or bare.startswith(_SYSTEM_SCHEMA_PREFIXES) + + +class ResolvedSchema(BaseModel): + """One schema in ingest scope. + + ``name`` is the *discovery* token, carried exactly as the dialect + enumerates it. ``explicit`` means the user named this single schema, so + its qualifier is written verbatim; a multi-schema request follows the + automatic rules instead, or listing the default schema alongside another + would re-qualify every model already on disk. + """ + + name: str | None = None + explicit: bool = False + is_default: bool = False + + +class IngestSchemaScope(BaseModel): + """The schemas one ingest pass will scan, plus what it left out.""" + + schemas: list[ResolvedSchema] = Field(default_factory=list) + # Non-system schemas NOT in scope. Hint only — populated when exactly one + # schema is in scope and ``all_schemas`` was not used. + other_schemas: list[str] = Field(default_factory=list) + # Schemas dropped because they belong to an attached catalog. Reported, + # never silently discarded. + skipped: list[SkippedTable] = Field(default_factory=list) + + +def _bare_schema(token: str | None) -> str: + """The last segment of a discovery token, i.e. the schema without catalog.""" + return token.rsplit(".", 1)[-1] if token else "" + + +def _matches_default(token: str, default_token: str | None) -> bool: + """Whether a user-supplied token names the connection's default schema. + + A qualified token must match in full — that is what stops ``aaa.main`` + and ``att_main.main`` both reading as the default. A bare token is + compared against the default's bare segment, so ``--schema main`` on + DuckDB still recognises ``fda.main``. + """ + if not default_token: + return False + if token == default_token: + return True + if "." in token: + return False + return token == _bare_schema(default_token) + + +def _current_catalog_only( + *, inspector: sa.engine.Inspector, tokens: list[str], +) -> tuple[list[str], list[SkippedTable]]: + """Restrict enumerated tokens to the connection's current catalog. + + ``--all-schemas`` means "this database", not "and everything anyone has + attached to the session" — DuckDB's ``get_schema_names()`` lists + ``ATTACH``ed catalogs too. Dropped tokens are reported with the exact + invocation that would ingest them. + """ + if not any("." in t for t in tokens): + return list(tokens), [] + from slayer.engine.introspect_utils import _current_catalog + + catalog = _current_catalog(inspector) + if not catalog: + return list(tokens), [] + kept: list[str] = [] + dropped: list[SkippedTable] = [] + for token in tokens: + token_catalog, _ = split_schema_token(token) + if token_catalog is None or token_catalog == catalog: + kept.append(token) + continue + dropped.append( + SkippedTable( + table_name=token, + reason=( + f"schema '{token}' belongs to attached catalog " + f"'{token_catalog}'; ingest it with --schema '{token}'" + ), + ) + ) + return kept, dropped + + +def _enumerate_schemas(inspector: sa.engine.Inspector) -> list[str]: + """Every non-system schema the connection can see, tokens as enumerated.""" + try: + names = list(inspector.get_schema_names() or []) + except Exception as exc: # noqa: BLE001 — enumeration is best-effort + logger.debug("get_schema_names failed: %s", exc) + return [] + return sorted(n for n in names if isinstance(n, str) and not _is_system_schema(n)) + + +def resolve_ingest_schemas( + *, + inspector: sa.engine.Inspector, + requested: list[str] | None, + all_schemas: bool, + datasource_schema: str | None, +) -> IngestSchemaScope: + """Decide which schemas one ingest pass covers. + + Precedence, first non-empty wins: ``all_schemas``; an explicit + ``requested`` list; the datasource's persisted ``schema_name``; the + connection's default schema. The persisted value is a *fallback*, + consulted only when nothing more specific was given — never a conflict. + """ + default_token = qualified_default_schema(inspector) + enumerated = _enumerate_schemas(inspector) + local, dropped = _current_catalog_only(inspector=inspector, tokens=enumerated) + + if all_schemas: + schemas = [ + ResolvedSchema( + name=token, + explicit=False, + is_default=(token == default_token), + ) + for token in local + ] + elif requested: + # A single named schema is honoured verbatim (``--schema public`` + # keeps producing ``public.orders``). A list is not: qualifying the + # default schema verbatim there would rewrite every model on disk. + single = len(requested) == 1 + schemas = [ + ResolvedSchema( + name=token, + explicit=single, + is_default=_matches_default(token, default_token), + ) + for token in requested + ] + elif datasource_schema: + schemas = [ + ResolvedSchema( + name=datasource_schema, + explicit=True, + is_default=_matches_default(datasource_schema, default_token), + ) + ] + else: + schemas = [ + ResolvedSchema(name=default_token, explicit=False, is_default=True) + ] + + other: list[str] = [] + if len(schemas) == 1 and not all_schemas: + in_scope = _bare_schema(schemas[0].name) + other = [b for b in (_bare_schema(t) for t in local) if b != in_scope] + return IngestSchemaScope( + schemas=schemas, other_schemas=other, skipped=dropped if all_schemas else [], + ) + + +def qualify_sql_table(*, obj: IngestableObject, resolved: ResolvedSchema) -> str: + """The ``sql_table`` value for ``obj``. + + Note this is NOT the discovery token: the catalog segment is dropped + because the connection's current catalog is already the right one, and + re-stating it would only break if the datasource were repointed. + + Default-schema objects stay unqualified so that widening the scan never + rewrites models that already exist. + """ + if resolved.explicit: + return f"{resolved.name}.{obj.name}" + if resolved.is_default or not resolved.name: + return obj.name + return f"{_bare_schema(resolved.name)}.{obj.name}" def _safe_object_names( @@ -849,10 +1091,19 @@ def list_ingestable_objects( ) -> list[IngestableObject]: """Discover every ingestable object in ``schema``, classified by kind. - Order is deterministic (tables, views, matviews) because - :func:`_assign_model_names` resolves collisions first-come. Deduped across + ``schema=None`` resolves to the connection's *default schema token* + rather than being passed through. On DuckDB a bare ``None`` returns + objects from every schema as bare names, which is how a model in a + non-default schema ended up with an unqualified ``sql_table``; and even + the bare default (``main``) still reaches into ``ATTACH``ed catalogs, + so the token has to carry the catalog. + + Order is deterministic (tables, views, matviews). Deduped across accessors — some dialects return views from ``get_table_names()``. """ + resolved_schema = ( + schema if schema is not None else qualified_default_schema(inspector) + ) objects: list[IngestableObject] = [] seen: set[str] = set() @@ -861,13 +1112,17 @@ def _add(names: list[str], kind: ObjectKind) -> None: if name in seen: continue seen.add(name) - objects.append(IngestableObject(name=name, kind=kind)) + objects.append( + IngestableObject(name=name, kind=kind, schema=resolved_schema) + ) - _add(list(inspector.get_table_names(schema=schema) or []), "table") + _add(list(inspector.get_table_names(schema=resolved_schema) or []), "table") if include_views: _add( _safe_object_names( - accessor_name="get_view_names", inspector=inspector, schema=schema + accessor_name="get_view_names", + inspector=inspector, + schema=resolved_schema, ), "view", ) @@ -875,60 +1130,113 @@ def _add(names: list[str], kind: ObjectKind) -> None: _safe_object_names( accessor_name="get_materialized_view_names", inspector=inspector, - schema=schema, + schema=resolved_schema, ), "materialized_view", ) return objects +def list_ingestable_objects_multi( + *, + inspector: sa.engine.Inspector, + scope: IngestSchemaScope, + include_views: bool = True, +) -> list[IngestableObject]: + """Discover objects across every schema in ``scope``, in scope order.""" + objects: list[IngestableObject] = [] + seen: set[tuple[str | None, str]] = set() + for resolved in scope.schemas: + for obj in list_ingestable_objects( + inspector=inspector, + schema=resolved.name, + include_views=include_views, + ): + key = (obj.schema, obj.name) + if key in seen: + continue + seen.add(key) + objects.append(obj) + return objects + + +def _spans_multiple_schemas(objects: list[IngestableObject]) -> bool: + """Whether ``objects`` were discovered in more than one schema.""" + return len({o.schema for o in objects}) > 1 + + +def _object_label(obj: IngestableObject, *, multi_schema: bool) -> str: + """How to name an object in a message — disambiguated only when needed.""" + if multi_schema and obj.schema: + return f"{_bare_schema(obj.schema)}.{obj.name}" + return obj.name + + def _assign_model_names( objects: list[IngestableObject], -) -> tuple[dict[str, str], list[SkippedTable]]: - """Map each object name to its model name, returning ``(mapping, skipped)``. - - Model names may not contain ``__`` (the SQL generator splits it back into a - join path, so ``a__b`` would silently query ``a -> b``); object names may, - so only the model name is sanitized. - - Unsanitized names are reserved first, so a real ``a_b`` beats a sanitized - ``a__b``. Collisions skip rather than suffix — suffixes shift with the - object set, orphaning models and churning drift. - - Both passes are scan-order independent. The sanitized pass walks its - candidates in sorted order, so when two objects collapse to the same name - (``a__b`` and ``a___b`` both yield ``a_b``) the winner is fixed by the name - itself, not by whichever the inspector happened to list first. Otherwise a - dialect changing its listing order would silently repoint the model at a - different physical object. + *, + resolved_by_schema: dict[str | None, ResolvedSchema] | None = None, +) -> tuple[dict[tuple[str | None, str], str], list[SkippedTable]]: + """Map each ``(schema, object name)`` to its model name. + + Returns ``(mapping, skipped)``. Model names may not contain ``__`` (the + SQL generator splits it back into a join path, so ``a__b`` would silently + query ``a -> b``); object names may, so only the model name is sanitized. + + Contention is resolved in ONE phase keyed on the final model name, not as + successive passes: two objects can now compete either because one was + sanitized into the other's name or because they live in different schemas, + and resolving those separately would leave the mixed case (``s1.a__b`` + sanitizing onto a real ``s2.a_b``) dependent on reservation order. + + The winner is fixed by a total order over the contenders, every key of + which is a property of the object itself — so the outcome is independent + of inspector listing order and of ``--schema`` argument order: + + 1. a name needing no sanitization beats one that did; + 2. then the default schema beats a non-default one; + 3. then the lower schema name; 4. then the lower object name. + + Losers are skipped, never suffixed — suffixes shift with the object set, + orphaning models and churning drift. """ - assigned: dict[str, str] = {} - taken: set[str] = {o.name for o in objects if "__" not in o.name} - skipped: list[SkippedTable] = [] + multi_schema = _spans_multiple_schemas(objects) + resolved_by_schema = resolved_by_schema or {} + + def _rank(obj: IngestableObject) -> tuple: + resolved = resolved_by_schema.get(obj.schema) + return ( + 0 if "__" not in obj.name else 1, + 0 if (resolved is not None and resolved.is_default) else 1, + _bare_schema(obj.schema), + obj.name, + ) + contenders: dict[str, list[IngestableObject]] = defaultdict(list) for obj in objects: - if "__" not in obj.name: - assigned[obj.name] = obj.name + model_name = ( + sanitize_model_name(obj.name) if "__" in obj.name else obj.name + ) + contenders[model_name].append(obj) - for obj in sorted( - (o for o in objects if "__" in o.name), key=lambda o: o.name - ): - candidate = sanitize_model_name(obj.name) - if candidate in taken: + assigned: dict[tuple[str | None, str], str] = {} + skipped: list[SkippedTable] = [] + for model_name in sorted(contenders): + winner, *losers = sorted(contenders[model_name], key=_rank) + assigned[(winner.schema, winner.name)] = model_name + winner_label = _object_label(winner, multi_schema=multi_schema) + for loser in losers: skipped.append( SkippedTable( - table_name=obj.name, - kind=obj.kind, + table_name=_object_label(loser, multi_schema=multi_schema), + kind=loser.kind, reason=( - f"name collision: sanitizing '__' yields " - f"'{candidate}', which is already taken" + f"name collision: model name '{model_name}' is also " + f"claimed by '{winner_label}'; ingest the schemas " + f"separately or use --exclude" ), ) ) - continue - taken.add(candidate) - assigned[obj.name] = candidate - return assigned, skipped @@ -943,7 +1251,7 @@ def _build_one_model( inspector: sa.engine.Inspector, obj: IngestableObject, model_name: str, - schema: str | None, + resolved: ResolvedSchema, data_source: str, fk_graph: dict[str, set[str]], has_cycles: bool, @@ -951,11 +1259,17 @@ def _build_one_model( table_set: set[str], ) -> SlayerModel: """Introspect one live object into a model. Raises on failure; the caller - isolates per-object.""" + isolates per-object. + + Introspection is driven by the *discovery* token (``resolved.name``) while + the emitted ``sql_table`` carries the qualifier — two different strings + that must not be conflated. + """ referenced = ( set() if has_cycles else _compute_transitive_closure(fk_graph, obj.name) ) - sql_table = f"{schema}.{obj.name}" if schema else obj.name + schema = resolved.name + sql_table = qualify_sql_table(obj=obj, resolved=resolved) model_joins = None if referenced: @@ -1036,82 +1350,224 @@ def _collect_fk_columns( return out +def _schema_hint(scope: IngestSchemaScope) -> str | None: + """Tell the user which schemas the scan left out, and how to get them. + + Eligibility is "exactly one schema in scope and others exist" — it is + deliberately independent of whether that schema was named explicitly. A + user who set ``schema_name`` months ago still needs to hear that another + schema has appeared since. + """ + if len(scope.schemas) != 1 or not scope.other_schemas: + return None + in_scope = _bare_schema(scope.schemas[0].name) + return ( + f"Note: ingested schema '{in_scope}' only. Other schemas in this " + f"datasource: {', '.join(scope.other_schemas)}.\n" + f"Re-run with --schema {scope.other_schemas[0]}, or --all-schemas, " + f"to ingest them." + ) + + +def _scan_one_schema( + *, + sa_engine: sa.Engine, + inspector: sa.engine.Inspector, + resolved: ResolvedSchema, + objects: list[IngestableObject], + name_by_object: dict[tuple[str | None, str], str], + data_source: str, + multi_schema: bool, +) -> tuple[list[SlayerModel], list[SkippedTable]]: + """Build every model for one schema. The FK graph is per-schema: joins + only ever resolve within the schema the objects were discovered in.""" + table_names = [o.name for o in objects] + table_set = set(table_names) + schema = resolved.name + + fk_graph = _build_fk_graph( + inspector=inspector, table_names=table_names, schema=schema + ) + has_cycles = False + try: + _check_acyclic(fk_graph) + except RollupGraphError as e: + logger.warning(f"FK graph has cycles, skipping rollup: {e}") + has_cycles = True + + fk_columns_by_table = _collect_fk_columns( + inspector=inspector, table_names=table_names, schema=schema + ) + + models: list[SlayerModel] = [] + skipped: list[SkippedTable] = [] + for obj in objects: + model_name = name_by_object.get((obj.schema, obj.name)) + if model_name is None: + continue # already recorded in ``skipped`` by _assign_model_names + try: + models.append( + _build_one_model( + sa_engine=sa_engine, + inspector=inspector, + obj=obj, + model_name=model_name, + resolved=resolved, + data_source=data_source, + fk_graph=fk_graph, + has_cycles=has_cycles, + fk_columns_by_table=fk_columns_by_table, + table_set=table_set, + ) + ) + except Exception as exc: # noqa: BLE001 — per-object isolation + logger.warning( + "Skipping %s %r in datasource %r: %s", + obj.kind, obj.name, data_source, exc, + ) + skipped.append( + SkippedTable( + table_name=_object_label(obj, multi_schema=multi_schema), + kind=obj.kind, + reason=str(exc), + ) + ) + return models, skipped + + +def _resolve_scope_args( + *, + schema: str | None, + schemas: list[str] | None, + all_schemas: bool, +) -> list[str] | None: + """Fold the three scope arguments into one requested list, or raise. + + Shared by every entry point — engine, CLI, REST and MCP — so the CLI's + mutually-exclusive group is not the only thing holding the line. + """ + if schema is not None and schemas is not None: + raise ValueError( + "Cannot set both 'schema' and 'schemas' — pass one or the other." + ) + if all_schemas and (schema is not None or schemas is not None): + raise ValueError( + "Cannot combine 'all_schemas' with an explicit schema — " + "'all_schemas' already covers every schema." + ) + if schemas is not None: + return list(schemas) or None + if schema is not None: + return [schema] + return None + + +def _default_schema_object_names( + *, + inspector: sa.engine.Inspector, + scope: IngestSchemaScope, + objects: list[IngestableObject], + include_views: bool, +) -> list[str]: + """Object names living in the connection's default schema. + + Derived from the objects already discovered when the default schema is in + scope; otherwise listed explicitly, which costs one extra catalog call on + the only path that needs it. + """ + default_token = qualified_default_schema(inspector) + if any(s.name == default_token for s in scope.schemas): + return [o.name for o in objects if o.schema == default_token] + try: + return [ + o.name + for o in list_ingestable_objects( + inspector=inspector, + schema=default_token, + include_views=include_views, + ) + ] + except Exception as exc: # noqa: BLE001 — the guard degrades, never aborts + logger.debug("default-schema listing failed: %s", exc) + return [] + + def ingest_datasource_report( datasource: DatasourceConfig, include_tables: list[str] | None = None, exclude_tables: list[str] | None = None, schema: str | None = None, include_views: bool = True, + schemas: list[str] | None = None, + all_schemas: bool = False, ) -> IngestionScanReport: """Introspect ``datasource``, returning models plus everything skipped. - Discovers views and matviews (``include_views``), and skips an unmodellable - object with a reason rather than aborting the run. + Scope resolution is :func:`resolve_ingest_schemas`; ``schema`` is kept as + a single-value alias for ``schemas``. Discovers views and matviews + (``include_views``), and skips an unmodellable object with a reason rather + than aborting the run. """ + requested = _resolve_scope_args( + schema=schema, schemas=schemas, all_schemas=all_schemas + ) from slayer.sql import engine_factory sa_engine = engine_factory.get_engine(datasource.resolve_env_vars()) try: inspector = sa.inspect(sa_engine) - objects = list_ingestable_objects( - inspector=inspector, schema=schema, include_views=include_views + scope = resolve_ingest_schemas( + inspector=inspector, + requested=requested, + all_schemas=all_schemas, + datasource_schema=datasource.schema_name or None, ) + discovered = list_ingestable_objects_multi( + inspector=inspector, scope=scope, include_views=include_views + ) + objects = discovered if include_tables: objects = [o for o in objects if o.name in include_tables] if exclude_tables: objects = [o for o in objects if o.name not in exclude_tables] - table_names = [o.name for o in objects] - table_set = set(table_names) - - name_by_object, skipped = _assign_model_names(objects) - - # Build FK graph, check for cycles - fk_graph = _build_fk_graph( - inspector=inspector, table_names=table_names, schema=schema + resolved_by_schema: dict[str | None, ResolvedSchema] = { + s.name: s for s in scope.schemas + } + name_by_object, skipped = _assign_model_names( + objects, resolved_by_schema=resolved_by_schema ) - has_cycles = False - try: - _check_acyclic(fk_graph) - except RollupGraphError as e: - logger.warning(f"FK graph has cycles, skipping rollup: {e}") - has_cycles = True + skipped = list(scope.skipped) + skipped + multi_schema = _spans_multiple_schemas(objects) - fk_columns_by_table = _collect_fk_columns( - inspector=inspector, table_names=table_names, schema=schema - ) - - models = [] - for obj in objects: - model_name = name_by_object.get(obj.name) - if model_name is None: - continue # already recorded in ``skipped`` by _assign_model_names - try: - models.append( - _build_one_model( - sa_engine=sa_engine, - inspector=inspector, - obj=obj, - model_name=model_name, - schema=schema, - data_source=datasource.name, - fk_graph=fk_graph, - has_cycles=has_cycles, - fk_columns_by_table=fk_columns_by_table, - table_set=table_set, - ) - ) - except Exception as exc: # noqa: BLE001 — per-object isolation - logger.warning( - "Skipping %s %r in datasource %r: %s", - obj.kind, obj.name, datasource.name, exc, - ) - skipped.append( - SkippedTable(table_name=obj.name, kind=obj.kind, reason=str(exc)) - ) + models: list[SlayerModel] = [] + for resolved in scope.schemas: + in_schema = [o for o in objects if o.schema == resolved.name] + if not in_schema: + continue + schema_models, schema_skips = _scan_one_schema( + sa_engine=sa_engine, + inspector=inspector, + resolved=resolved, + objects=in_schema, + name_by_object=name_by_object, + data_source=datasource.name, + multi_schema=multi_schema, + ) + models.extend(schema_models) + skipped.extend(schema_skips) return IngestionScanReport( - models=models, skipped=skipped, objects=objects + models=models, + skipped=skipped, + objects=objects, + schema_hint=_schema_hint(scope), + default_schema_objects=_default_schema_object_names( + inspector=inspector, + scope=scope, + objects=discovered, + include_views=include_views, + ), ) finally: # One-shot admin operation, not a hot query path. Disposing releases @@ -1130,6 +1586,8 @@ def ingest_datasource( exclude_tables: list[str] | None = None, schema: str | None = None, include_views: bool = True, + schemas: list[str] | None = None, + all_schemas: bool = False, ) -> list[SlayerModel]: """Models only, for callers that don't need the skip report.""" return ingest_datasource_report( @@ -1138,6 +1596,8 @@ def ingest_datasource( exclude_tables=exclude_tables, schema=schema, include_views=include_views, + schemas=schemas, + all_schemas=all_schemas, ).models @@ -1250,6 +1710,9 @@ class AdditiveMergeResult(BaseModel): new_joins: list[str] = Field(default_factory=list) widened_columns: list[str] = Field(default_factory=list) kind_changed: bool = False + # ``"reports → openfda_rest.reports"`` when a missing schema qualifier was + # repaired, else None. + sql_table_change: str | None = None def _additive_merge_existing( @@ -1280,6 +1743,10 @@ def _additive_merge_existing( usually changes no columns at all. A field that never refreshed would confidently lie about precisely the case it was added for. A ``None`` from a path that doesn't classify never erases a known value. + * Carve-out: a MISSING ``sql_table`` schema qualifier is repaired, so a + model ingested before schemas were recorded becomes queryable again on + the next run. An existing qualifier is never rewritten — healing adds + one, it does not repoint a model someone deliberately aimed elsewhere. """ existing_by_name: dict[str, Column] = {c.name: c for c in persisted.columns} fresh_by_name: dict[str, Column] = {c.name: c for c in fresh.columns} @@ -1314,17 +1781,25 @@ def _additive_merge_existing( and fresh.source_kind != persisted.source_kind ) + # A qualifier repair typically changes nothing else either, so it has to + # participate in the short-circuit for the same reason ``kind_changed`` + # does — otherwise the repaired model is computed and then discarded. + sql_table_change = _qualifier_repair(persisted=persisted, fresh=fresh) + if not ( new_column_names or new_join_targets or widened_column_names or kind_changed + or sql_table_change ): return AdditiveMergeResult(merged=persisted) update: dict[str, Any] = {"columns": merged_columns, "joins": new_joins} if kind_changed: update["source_kind"] = fresh.source_kind + if sql_table_change: + update["sql_table"] = fresh.sql_table return AdditiveMergeResult( merged=persisted.model_copy(update=update), @@ -1332,6 +1807,76 @@ def _additive_merge_existing( new_joins=new_join_targets, widened_columns=widened_column_names, kind_changed=kind_changed, + sql_table_change=sql_table_change, + ) + + +def _qualifier_repair( + *, persisted: SlayerModel, fresh: SlayerModel, +) -> str | None: + """``"before → after"`` when ``fresh`` supplies a qualifier ``persisted`` + is missing, else ``None``. + + Keyed on the bare object names matching: ``reports`` and ``s.other`` are + unrelated tables that happen to share a model name, not a repair. + """ + before, after = persisted.sql_table, fresh.sql_table + if not before or not after: + return None + if "." in before or "." not in after: + return None + if _bare_table_name(after) != before: + return None + return f"{before} → {after}" + + +class ProcessTableOutcome(BaseModel): + """What the additive pass did with one freshly-introspected model. + + A skip ("I declined this one") is not an error ("this failed to persist"), + so it travels separately and is reported separately. + """ + + addition: Any | None = None + skipped: SkippedTable | None = None + + +def _cross_schema_conflict( + *, + model_name: str, + persisted: SlayerModel, + fresh: SlayerModel, + default_schema_objects: set[str], +) -> SkippedTable | None: + """Refuse to merge two different schemas' tables into one model. + + Two sequential single-schema ingests would otherwise fuse them, with no + flag involved. The second check covers the case a schema comparison alone + cannot: a default-schema model is persisted *unqualified*, so a fresh + qualified object with the same bare name looks like a repair when it is + actually a different table. + """ + persisted_table = persisted.sql_table or "" + fresh_table = fresh.sql_table or "" + persisted_schema = _schema_of(persisted_table) + fresh_schema = _schema_of(fresh_table) + if not fresh_schema: + return None + + conflicting = persisted_schema is not None and persisted_schema != fresh_schema + shadows_default = ( + persisted_schema is None and persisted_table in default_schema_objects + ) + if not (conflicting or shadows_default): + return None + return SkippedTable( + table_name=fresh_table, + kind=fresh.source_kind, + reason=( + f"cross-schema conflict: model '{model_name}' is bound to " + f"'{persisted_table}', and '{fresh_table}' is a different table; " + f"ingest the schemas separately or use --exclude" + ), ) @@ -1341,62 +1886,89 @@ async def _process_one_table( fresh: SlayerModel, datasource: DatasourceConfig, storage: StorageBackend, -): - """Save / merge one freshly-introspected model, returning the - ``ModelAddition`` to record. Raises on persistence failure — the caller - isolates errors per-model. + default_schema_objects: set[str] | None = None, +) -> ProcessTableOutcome: + """Save / merge one freshly-introspected model. Raises on persistence + failure — the caller isolates errors per-model. """ from slayer.engine.schema_drift import ModelAddition persisted = await storage.get_model(table_name, data_source=datasource.name) if persisted is None: await storage.save_model(fresh) - return ModelAddition( - model_name=table_name, - data_source=datasource.name, - created=True, - new_columns=[c.name for c in fresh.columns], - new_joins=[j.target_model for j in fresh.joins], - source_kind=fresh.source_kind, + return ProcessTableOutcome( + addition=ModelAddition( + model_name=table_name, + data_source=datasource.name, + created=True, + new_columns=[c.name for c in fresh.columns], + new_joins=[j.target_model for j in fresh.joins], + source_kind=fresh.source_kind, + ) ) if persisted.sql or persisted.source_queries: # User-authored sql / query-backed model with the matching name — # leave it alone. - return None + return ProcessTableOutcome() + + conflict = _cross_schema_conflict( + model_name=table_name, + persisted=persisted, + fresh=fresh, + default_schema_objects=default_schema_objects or set(), + ) + if conflict is not None: + return ProcessTableOutcome(skipped=conflict) + outcome = _additive_merge_existing( persisted=persisted, fresh=fresh, sqlite_widen_enabled=(datasource.type or "").lower() == "sqlite", ) - # ``kind_changed`` must gate the save too: a view→table flip - # usually adds no columns and no joins, so without it the refreshed model - # would be computed and then thrown away. + # ``kind_changed`` and ``sql_table_change`` must gate the save too: a + # view→table flip or a qualifier repair usually adds no columns and no + # joins, so without them the refreshed model would be computed and then + # thrown away. if ( outcome.new_columns or outcome.new_joins or outcome.widened_columns or outcome.kind_changed + or outcome.sql_table_change ): await storage.save_model(outcome.merged) kind_change = None if outcome.kind_changed: before = persisted.source_kind or "unknown" kind_change = f"{before} → {fresh.source_kind}" - return ModelAddition( - model_name=table_name, - data_source=datasource.name, - created=False, - new_columns=outcome.new_columns, - new_joins=outcome.new_joins, - widened_columns=outcome.widened_columns, - source_kind=outcome.merged.source_kind, - kind_change=kind_change, + return ProcessTableOutcome( + addition=ModelAddition( + model_name=table_name, + data_source=datasource.name, + created=False, + new_columns=outcome.new_columns, + new_joins=outcome.new_joins, + widened_columns=outcome.widened_columns, + source_kind=outcome.merged.source_kind, + kind_change=kind_change, + sql_table_change=outcome.sql_table_change, + ) ) def _bare_table_name(sql_table: str) -> str: """Strip an optional schema prefix from a ``schema.table`` reference.""" - return sql_table.split(".", 1)[1] if "." in sql_table else sql_table + return split_sql_table(sql_table)[1] + + +def _schema_of(sql_table: str) -> str | None: + """The bare schema segment of a ``sql_table``, or None when unqualified. + + Built on :func:`split_sql_table` so there is one dotted-name parser, not + two that disagree about three-part names. + """ + schema_token, _ = split_sql_table(sql_table) + return _bare_schema(schema_token) if schema_token else None async def _scoped_models_for_validation( @@ -1435,6 +2007,8 @@ async def ingest_datasource_idempotent( exclude_tables: list[str] | None = None, schema: str | None = None, include_views: bool = True, + schemas: list[str] | None = None, + all_schemas: bool = False, ): """Idempotent re-ingestion. @@ -1460,6 +2034,8 @@ async def ingest_datasource_idempotent( additions: list[ModelAddition] = [] errors: list[IngestionError] = [] + # Rejected before the scan so a conflicting call never opens a connection. + _resolve_scope_args(schema=schema, schemas=schemas, all_schemas=all_schemas) # ``ingest_datasource_report`` is sync (it drives SQLAlchemy ``Inspector``). # Offload to a thread so a slow / large datasource doesn't block the @@ -1471,8 +2047,12 @@ async def ingest_datasource_idempotent( exclude_tables=exclude_tables, schema=schema, include_views=include_views, + schemas=schemas, + all_schemas=all_schemas, ) fresh_models = scan.models + skipped = list(scan.skipped) + default_schema_objects = set(scan.default_schema_objects) fresh_by_name = {m.name: m for m in fresh_models} # Keyed on the LIVE OBJECT name, not the model name. ``_scoped_models_for_validation`` # compares this against ``_bare_table_name(m.sql_table)``, so using model @@ -1485,14 +2065,17 @@ async def ingest_datasource_idempotent( for table_name, fresh in fresh_by_name.items(): try: - addition = await _process_one_table( + outcome = await _process_one_table( table_name=table_name, fresh=fresh, datasource=datasource, storage=storage, + default_schema_objects=default_schema_objects, ) - if addition is not None: - additions.append(addition) + if outcome.addition is not None: + additions.append(outcome.addition) + if outcome.skipped is not None: + skipped.append(outcome.skipped) except Exception as exc: # noqa: BLE001 — best-effort per-model isolation errors.append( IngestionError( @@ -1545,8 +2128,9 @@ async def ingest_datasource_idempotent( additions=additions, to_delete=list(to_delete), errors=errors, - skipped=scan.skipped, + skipped=skipped, objects=scan.objects, + schema_hint=scan.schema_hint, ) @@ -1639,7 +2223,16 @@ def _print_ingest_addition( return widened = getattr(addition, "widened_columns", []) or [] kind_change = getattr(addition, "kind_change", None) - if not (addition.new_columns or addition.new_joins or widened or kind_change): + # A qualifier repair adds no columns, so without it in the gate the whole + # line — the point of the re-ingest — would print nothing at all. + sql_table_change = getattr(addition, "sql_table_change", None) + if not ( + addition.new_columns + or addition.new_joins + or widened + or kind_change + or sql_table_change + ): return details = [] if addition.new_columns: @@ -1650,6 +2243,8 @@ def _print_ingest_addition( details.append(f"widened: {', '.join(widened)}") if kind_change: details.append(f"source_kind: {kind_change}") + if sql_table_change: + details.append(f"sql_table: {sql_table_change}") print(f"Updated: {addition.model_name} ({'; '.join(details)})", file=out) diff --git a/slayer/engine/introspect_utils.py b/slayer/engine/introspect_utils.py index 17ea153d..7d450b7a 100644 --- a/slayer/engine/introspect_utils.py +++ b/slayer/engine/introspect_utils.py @@ -13,7 +13,8 @@ from __future__ import annotations -from typing import Dict, List, Optional +from pathlib import Path +from typing import Any, Dict, List, Optional import sqlalchemy as sa @@ -70,52 +71,200 @@ def _parse_info_schema_is_float(data_type_str: str) -> bool: return True # No precision/scale info, default to float +def split_sql_table(sql_table: str) -> tuple[Optional[str], str]: + """Split a ``sql_table`` reference into ``(schema_token, object_name)``. + + The schema token is everything before the FINAL dot, catalog segment + included — ``proj.dataset.tbl`` yields ``("proj.dataset", "tbl")``. + Truncating it would introspect the wrong catalog, or (on DuckDB) match + nothing at all. + """ + schema_token, sep, obj = sql_table.rpartition(".") + if not sep: + return None, sql_table + return (schema_token or None), obj + + +def split_schema_token(token: str) -> tuple[Optional[str], str]: + """Split a discovery token into ``(catalog, schema)``. + + Discovery tokens are at most ``catalog.schema``, so this splits on the + FIRST dot — the mirror image of :func:`split_sql_table`. + """ + catalog, sep, schema = token.partition(".") + if not sep: + return None, token + return (catalog or None), schema + + +_UNSET = object() +_DEFAULT_SCHEMA_ATTR = "_slayer_default_schema_token" + + +def _current_catalog(inspector: sa.engine.Inspector) -> Optional[str]: + """The catalog (database) the connection is currently attached to.""" + try: + with inspector.engine.connect() as conn: + catalog = conn.exec_driver_sql("SELECT current_database()").scalar() + if isinstance(catalog, str) and catalog: + return catalog + except Exception: # noqa: BLE001 — probe only; the URL stem is the fallback + pass + try: + database = inspector.engine.url.database + except Exception: # noqa: BLE001 + return None + return Path(database).stem if database else None + + +def qualified_default_schema( + inspector: sa.engine.Inspector, +) -> Optional[str]: + """The discovery token identifying the connection's default schema. + + DuckDB's ``get_schema_names()`` always returns catalog-qualified tokens + (``fda.main``), and the qualified form is the *safe* one: a bare ``main`` + makes ``get_table_names`` and ``has_table`` reach into ``ATTACH``ed + catalogs. So the token is matched against what the dialect actually + enumerates, and returned in exactly that shape. Dialects that enumerate + bare names get their bare default back unchanged. + + Returns ``None`` when the dialect reports no default schema, which callers + treat as "pass ``None`` to the accessors", i.e. today's behaviour. + """ + cached = getattr(inspector, _DEFAULT_SCHEMA_ATTR, _UNSET) + if cached is not _UNSET: + return cached if isinstance(cached, str) else None + token = _compute_default_schema_token(inspector) + try: + setattr(inspector, _DEFAULT_SCHEMA_ATTR, token) + except Exception: # noqa: BLE001 — caching is an optimisation, not a contract + pass + return token + + +def _compute_default_schema_token( + inspector: sa.engine.Inspector, +) -> Optional[str]: + try: + default = inspector.default_schema_name + except Exception: # noqa: BLE001 — a dialect may not implement it + return None + if not isinstance(default, str) or not default: + return None + try: + names = list(inspector.get_schema_names() or []) + except Exception: # noqa: BLE001 — enumeration is best-effort + return default + if default in names: + return default + catalog = _current_catalog(inspector) + if catalog and f"{catalog}.{default}" in names: + return f"{catalog}.{default}" + # A dialect that qualifies its tokens but whose current catalog we could + # not determine: accept a unique last-segment match, never an ambiguous one. + matches = [n for n in names if n.rsplit(".", 1)[-1] == default] + return matches[0] if len(matches) == 1 else default + + +def _info_schema_column(col_name: str, data_type_str: str) -> Dict: + """Map one ``INFORMATION_SCHEMA.columns`` row to SLayer's column shape.""" + # Strip precision info (e.g. "DECIMAL(10,2)" → "DECIMAL") + base_type = data_type_str.split("(")[0].upper().strip() + sa_type = _INFO_SCHEMA_TYPE_MAP.get(base_type) + is_float = base_type in _FLOAT_LIKE_INFO_SCHEMA_TYPES + # NUMERIC/DECIMAL: check scale to decide float vs integer + if base_type in ("NUMERIC", "DECIMAL") or ( + sa_type is None and ("DECIMAL" in base_type or "NUMERIC" in base_type) + ): + sa_type = sa_type or DataType.DOUBLE + is_float = _parse_info_schema_is_float(data_type_str) + elif sa_type is None and "INT" in base_type: + # DEV-1361: integer-shaped types should narrow to INT, not the + # coarse DOUBLE fallback (e.g. MEDIUMINT, TINYINT variants not + # otherwise mapped). + sa_type = DataType.INT + elif sa_type is None and ("CHAR" in base_type or "TEXT" in base_type): + sa_type = DataType.TEXT + return {"name": col_name, "type": sa_type or DataType.TEXT, "is_float": is_float} + + +def _columns_in_schema( + sa_engine: sa.Engine, table_name: str, schema: str, +) -> List[Dict]: + """Columns of ``schema.table_name``, filtered on the catalog too. + + ``information_schema.columns`` carries ``table_catalog``, and it is + populated for ``ATTACH``ed catalogs — so a catalog-qualified token filters + exactly, instead of matching nothing (which silently produced a + column-less model) or matching everywhere (which unioned two tables' + columns together). + """ + catalog, bare_schema = split_schema_token(schema) + clauses = ["table_name = :table_name", "table_schema = :schema"] + params: Dict[str, Any] = {"table_name": table_name, "schema": bare_schema} + if catalog is not None: + clauses.append("table_catalog = :catalog") + params["catalog"] = catalog + sql = ( + "SELECT column_name, data_type " + "FROM information_schema.columns " + "WHERE " + " AND ".join(clauses) + " " + "ORDER BY ordinal_position" + ) + with sa_engine.connect() as conn: + rows = conn.execute(sa.text(sql), params).fetchall() + return [_info_schema_column(name, type_str) for name, type_str in rows] + + def _get_columns_fallback( sa_engine: sa.Engine, table_name: str, schema: Optional[str], + *, + default_schema: Optional[str] = None, ) -> List[Dict]: - """Get columns via INFORMATION_SCHEMA when Inspector.get_columns() fails.""" + """Get columns via INFORMATION_SCHEMA when Inspector.get_columns() fails. + + On DuckDB this is not a rare backstop — ``Inspector.get_columns`` raises + for every schema — so it is the primary column path for a Tier-1 dialect. + + With no ``schema`` the query cannot be narrowed, so same-named tables in + two schemas both match. Rather than unioning their columns (which produced + models referencing columns their table does not have) the rows are grouped + by catalog+schema: one group is used, ``default_schema`` breaks a tie, and + anything still ambiguous raises. Picking a winner by sort order would + replace union corruption with wrong-table corruption, which is harder to + notice; the caller isolates the raise per object and reports a skip. + """ if schema: - sql = ( - "SELECT column_name, data_type " - "FROM information_schema.columns " - "WHERE table_name = :table_name " - "AND table_schema = :schema " - "ORDER BY ordinal_position" - ) - params = {"table_name": table_name, "schema": schema} - else: - sql = ( - "SELECT column_name, data_type " - "FROM information_schema.columns " - "WHERE table_name = :table_name " - "ORDER BY ordinal_position" - ) - params = {"table_name": table_name} + return _columns_in_schema(sa_engine, table_name, schema) + + sql = ( + "SELECT table_catalog, table_schema, column_name, data_type " + "FROM information_schema.columns " + "WHERE table_name = :table_name " + "ORDER BY table_catalog, table_schema, ordinal_position" + ) with sa_engine.connect() as conn: - rows = conn.execute(sa.text(sql), params).fetchall() - result = [] - for col_name, data_type_str in rows: - # Strip precision info (e.g. "DECIMAL(10,2)" → "DECIMAL") - base_type = data_type_str.split("(")[0].upper().strip() - sa_type = _INFO_SCHEMA_TYPE_MAP.get(base_type) - is_float = base_type in _FLOAT_LIKE_INFO_SCHEMA_TYPES - # NUMERIC/DECIMAL: check scale to decide float vs integer - if base_type in ("NUMERIC", "DECIMAL") or ( - sa_type is None and ("DECIMAL" in base_type or "NUMERIC" in base_type) - ): - sa_type = sa_type or DataType.DOUBLE - is_float = _parse_info_schema_is_float(data_type_str) - elif sa_type is None and "INT" in base_type: - # DEV-1361: integer-shaped types should narrow to INT, not the - # coarse DOUBLE fallback (e.g. MEDIUMINT, TINYINT variants not - # otherwise mapped). - sa_type = DataType.INT - elif sa_type is None and ("CHAR" in base_type or "TEXT" in base_type): - sa_type = DataType.TEXT - result.append({"name": col_name, "type": sa_type or DataType.TEXT, "is_float": is_float}) - return result + rows = conn.execute(sa.text(sql), {"table_name": table_name}).fetchall() + + by_token: Dict[str, List[Dict]] = {} + for catalog, schema_name, col_name, data_type_str in rows: + token = f"{catalog}.{schema_name}" if catalog else schema_name + by_token.setdefault(token, []).append( + _info_schema_column(col_name, data_type_str) + ) + if not by_token: + return [] + if len(by_token) == 1: + return next(iter(by_token.values())) + if default_schema in by_token: + return by_token[default_schema] + raise ValueError( + f"Column lookup for {table_name!r} is ambiguous: it exists in " + f"{', '.join(sorted(by_token))}. Pass an explicit schema." + ) def _safe_get_columns( @@ -124,8 +273,19 @@ def _safe_get_columns( table_name: str, schema: Optional[str], ) -> List[Dict]: - """Get columns, falling back to INFORMATION_SCHEMA on failure.""" + """Get columns, falling back to INFORMATION_SCHEMA on failure. + + Resolves ``schema=None`` to the connection's default schema token before + falling back, so the schema-blind query never runs for a dialect that + reports a default. + """ try: return inspector.get_columns(table_name, schema=schema) except Exception: - return _get_columns_fallback(sa_engine, table_name, schema) + default_token = qualified_default_schema(inspector) + return _get_columns_fallback( + sa_engine, + table_name, + schema if schema is not None else default_token, + default_schema=default_token, + ) diff --git a/slayer/engine/schema_drift.py b/slayer/engine/schema_drift.py index 5e1478cb..f0339077 100644 --- a/slayer/engine/schema_drift.py +++ b/slayer/engine/schema_drift.py @@ -15,6 +15,7 @@ import asyncio import logging +from collections import Counter from typing import ( Annotated, Any, @@ -42,7 +43,7 @@ ) from slayer.core.query import SlayerQuery from slayer.sql.sql_predicate import parse_sql_predicate -from slayer.engine.introspect_utils import _safe_get_columns +from slayer.engine.introspect_utils import _safe_get_columns, split_sql_table from slayer.engine.ingestion import ( _safe_get_pk_constraint, _sa_type_is_float, @@ -117,6 +118,9 @@ class ModelAddition(BaseModel): # Human-readable transition (e.g. "view → table") when a re-ingest found # the live object had changed kind. None when nothing changed. kind_change: str | None = None + # Human-readable transition (e.g. "reports → openfda_rest.reports") when a + # re-ingest repaired a missing schema qualifier. None when nothing changed. + sql_table_change: str | None = None class IngestionError(BaseModel): @@ -143,6 +147,10 @@ class IdempotentIngestResult(BaseModel): # circular import with ``engine.ingestion``; runtime entries are # ``IngestableObject``. objects: list[Any] = Field(default_factory=list) + # Set when exactly one schema was scanned and the datasource holds others. + # Travels in the response body so REST / MCP callers see the same nudge + # the CLI prints. Advisory — never an error. + schema_hint: str | None = None class AppliedEntry(BaseModel): @@ -1655,14 +1663,36 @@ def compute_datasource_drops( # =========================================================================== +def _alias_keys(schema_token: str | None, name: str) -> list[str]: + """Progressively shorter lookup keys for one live object, longest first. + + A model written before the catalog was known says ``schema.table``; one + written before schemas were recorded at all says just ``table``. Both must + keep resolving, so both aliases are offered — but only inserted when they + are unambiguous across everything scanned. + """ + if not schema_token: + return [name] + return [f"{schema_token.rsplit('.', 1)[-1]}.{name}", name] + + def _live_schema_for_datasource( *, datasource: DatasourceConfig, - schema: str | None = None, + schemas: list[str | None] | None = None, ) -> dict[str, LiveTable]: - """Return ``{object_name: LiveTable}`` for every live table AND view in - the DS, using SQLAlchemy ``Inspector`` and the same fallback path as - auto-ingestion (``slayer/engine/ingestion.py``). + """Return ``{object_key: LiveTable}`` for every live table AND view in the + listed schemas, using SQLAlchemy ``Inspector`` and the same fallback path + as auto-ingestion (``slayer/engine/ingestion.py``). + + Keys are the FULL discovery identity ``"."``, + catalog segment included, plus shorter aliases inserted only when unique + across everything scanned. On a clash the ambiguous alias is dropped, so + a lookup misses rather than silently resolving to another catalog's + same-named table. + + ``schemas=None`` means the connection's default schema, matching the + previous single-schema signature. Views are included **unconditionally** — there is deliberately no ``include_views`` parameter here, and adding one would be a bug. @@ -1681,29 +1711,43 @@ def _live_schema_for_datasource( sa_engine = engine_factory.get_engine(datasource.resolve_env_vars()) try: inspector = sa.inspect(sa_engine) - table_names = [ - o.name - for o in list_ingestable_objects( + entries: list[tuple[str | None, str, LiveTable]] = [] + for schema in schemas if schemas is not None else [None]: + for obj in list_ingestable_objects( inspector=inspector, schema=schema, include_views=True - ) - ] + ): + try: + entries.append(( + obj.schema, + obj.name, + _introspect_one_table( + inspector=inspector, + sa_engine=sa_engine, + table_name=obj.name, + schema=obj.schema, + ), + )) + except Exception as exc: + logger.warning( + "validate_models: failed to introspect %r in datasource " + "%r: %s", + obj.name, + datasource.name, + exc, + ) + out: dict[str, LiveTable] = {} - for table_name in table_names: - try: - out[table_name] = _introspect_one_table( - inspector=inspector, - sa_engine=sa_engine, - table_name=table_name, - schema=schema, - ) - except Exception as exc: - logger.warning( - "validate_models: failed to introspect %r in datasource " - "%r: %s", - table_name, - datasource.name, - exc, - ) + for schema_token, name, live in entries: + out[f"{schema_token}.{name}" if schema_token else name] = live + alias_counts: Counter[str] = Counter( + alias + for schema_token, name, _ in entries + for alias in _alias_keys(schema_token, name) + ) + for schema_token, name, live in entries: + for alias in _alias_keys(schema_token, name): + if alias_counts[alias] == 1 and alias not in out: + out[alias] = live return out finally: # Same rationale as ``ingest_datasource``: this is a one-shot @@ -1822,14 +1866,22 @@ def _strip_ident_quotes(ident: str) -> str: def _resolve_live_table( *, sql_table: str, live_tables: dict[str, LiveTable] ) -> LiveTable | None: - """Look up a model's ``sql_table`` in the live introspection map, - falling back to the bare name when the persisted value is schema- - qualified (``schema.table``) and unquoting double-quoted identifiers - (e.g. ``prod."Company"`` for case-sensitive Postgres tables). + """Look up a model's ``sql_table`` in the live introspection map, walking + progressively shorter keys and unquoting double-quoted identifiers (e.g. + ``prod."Company"`` for case-sensitive Postgres tables). + + Order matters: full identity, then the last two segments, then the bare + name. The live map drops ambiguous short keys, so a miss on a shorter + candidate means "this could be either table" — and returning None there + is correct, where returning an arbitrary match would diff a model against + the wrong table. """ candidates = [sql_table] - if "." in sql_table: - candidates.append(sql_table.split(".", 1)[1]) + parts = sql_table.split(".") + if len(parts) > 2: + candidates.append(".".join(parts[-2:])) + if len(parts) > 1: + candidates.append(parts[-1]) # Materialise the snapshot before extending — a bare generator # ``(_strip_ident_quotes(c) for c in candidates)`` would iterate the # list lazily WHILE ``extend`` appends to it, so every appended item @@ -2060,13 +2112,22 @@ async def _collect_sql_table_diffs( """ if not sql_table_models: return {} - # Honour the datasource's configured schema_name so non-default-schema - # datasources diff against the right table set; otherwise SQLAlchemy - # introspects the default and produces false WholeModelDeletes. + # The schema set is derived from the models being validated, plus the + # datasource's configured default. Introspecting only the default schema + # made every non-default-schema model unresolvable, and an unresolvable + # model is a ``WholeModelDelete`` that ``validate-models --force-clean`` + # acts on — so getting this set wrong is a data-loss path, not a + # false-positive nuisance. + schemas: set[str | None] = { + split_sql_table(m.sql_table)[0] for m in sql_table_models if m.sql_table + } + schemas.discard(None) + schemas.add(datasource.schema_name or None) live_tables = await asyncio.to_thread( _live_schema_for_datasource, datasource=datasource, - schema=datasource.schema_name or None, + # ``None`` (the connection default) first, then a stable order. + schemas=sorted(schemas, key=lambda s: (s is not None, s or "")), ) probe_drifts_by_model = await _sqlite_probe_drifts_for_models( datasource=datasource, diff --git a/slayer/mcp/server.py b/slayer/mcp/server.py index 8c45360b..90e570c3 100644 --- a/slayer/mcp/server.py +++ b/slayer/mcp/server.py @@ -115,6 +115,16 @@ def _fetch_tables( return None, str(e) +def _csv_arg(value: str) -> list[str] | None: + """Split a comma-separated tool argument, or None when it is empty. + + MCP tool arguments are flat scalars, so lists travel as CSV — matching + ``include_tables``' existing style rather than adding a second convention. + """ + items = [part.strip() for part in (value or "").split(",")] + return [item for item in items if item] or None + + def _empty_ingest_message(*, schema_name: str, ds: DatasourceConfig) -> str: """Agent-facing wrapper over the shared engine renderer.""" return _shared_empty_ingest_message( @@ -1311,6 +1321,8 @@ async def create_datasource( connection_string: str | None = None, schema_name: str | None = None, auto_ingest: bool = True, + schemas: str = "", + all_schemas: bool = False, ) -> str: """Create a database connection, verify it, and auto-ingest models. Use ${ENV_VAR} syntax in credentials to reference environment variables. @@ -1323,13 +1335,26 @@ async def create_datasource( username: Database username. password: Database password. connection_string: Full connection string as alternative to individual fields. - schema_name: Default schema name. Also used as the schema for auto-ingestion. + schema_name: Default schema name. Persisted, and used as the schema for auto-ingestion. auto_ingest: Automatically ingest models from the database schema (default: true). Set to false to skip. + schemas: Comma-separated schemas to ingest. Alternative to schema_name; do not set both. Not persisted. + all_schemas: Ingest every non-system schema in the database. Do not combine with the other two. Example: create_datasource(name="mydb", type="postgres", host="localhost", port=5432, database="app", username="user", password="pass") """ + from slayer.engine.ingestion import _resolve_scope_args from slayer.engine.ingestion import ingest_datasource as _ingest + schema_list = _csv_arg(schemas) + try: + _resolve_scope_args( + schema=schema_name, + schemas=schema_list, + all_schemas=all_schemas, + ) + except ValueError as exc: + return f"Cannot create datasource: {exc}" + data = _build_dict( name=name, type=type, @@ -1358,9 +1383,15 @@ async def create_datasource( if not auto_ingest: return "\n".join(lines) - # Auto-ingest models + # Auto-ingest models. The scope is passed explicitly rather than left + # to the persisted ``schema_name``, so the two can never be read as a + # conflict. try: - models = _ingest(datasource=ds, schema=schema_name or None) + models = _ingest( + datasource=ds, + schemas=schema_list or ([schema_name] if schema_name else None), + all_schemas=all_schemas, + ) except Exception as e: if isinstance(e, (sa.exc.OperationalError, sa.exc.DatabaseError)): lines.append(f"Auto-ingestion failed: {_friendly_db_error(e)}") @@ -1380,9 +1411,9 @@ async def create_datasource( if not models and not save_errors: lines.append("No tables found to ingest.") - schemas = _get_schemas(ds) - if schemas: - lines.append(f"Available schemas: {', '.join(schemas)}") + available = _get_schemas(ds) + if available: + lines.append(f"Available schemas: {', '.join(available)}") elif models: lines.append(f"Ingested {len(models)} model(s):") for m in models: @@ -1659,7 +1690,13 @@ async def delete_datasource(name: str) -> str: # ----------------------------------------------------------------------- @mcp.tool() - async def ingest_datasource_models(datasource_name: str, include_tables: str = "", schema_name: str = "") -> str: + async def ingest_datasource_models( + datasource_name: str, + include_tables: str = "", + schema_name: str = "", + schemas: str = "", + all_schemas: bool = False, + ) -> str: """Auto-discover tables in a database and create / additively update semantic models from them. Idempotent (DEV-1356): re-runs are additive only. New columns and joins @@ -1670,21 +1707,38 @@ async def ingest_datasource_models(datasource_name: str, include_tables: str = " Args: datasource_name: Name of an existing datasource (from list_datasources). include_tables: Comma-separated list of table names to include. If empty, all tables are ingested. - schema_name: Database schema to inspect (e.g. "public"). If empty, uses the default schema. + schema_name: A single database schema to inspect (e.g. "public"). If empty, uses the default schema. + schemas: Comma-separated schemas to inspect. Alternative to schema_name; do not set both. + all_schemas: Inspect every non-system schema in the current database. Do not combine with the other two. """ - from slayer.engine.ingestion import ingest_datasource_idempotent + from slayer.engine.ingestion import ( + _resolve_scope_args, + ingest_datasource_idempotent, + ) ds = await storage.get_datasource(datasource_name) if ds is None: return f"Datasource '{datasource_name}' not found." + schema_list = _csv_arg(schemas) + try: + _resolve_scope_args( + schema=schema_name or None, + schemas=schema_list, + all_schemas=all_schemas, + ) + except ValueError as exc: + return f"Cannot ingest: {exc}" + try: - include = [t.strip() for t in include_tables.split(",") if t.strip()] or None + include = _csv_arg(include_tables) result = await ingest_datasource_idempotent( datasource=ds, storage=storage, include_tables=include, schema=schema_name or None, + schemas=schema_list, + all_schemas=all_schemas, ) except Exception as e: if isinstance(e, (sa.exc.OperationalError, sa.exc.DatabaseError)): diff --git a/slayer/storage/type_refinement.py b/slayer/storage/type_refinement.py index eae213da..aa15f0d3 100644 --- a/slayer/storage/type_refinement.py +++ b/slayer/storage/type_refinement.py @@ -30,6 +30,7 @@ from slayer.core.enums import DataType from slayer.core.models import DatasourceConfig +from slayer.engine.introspect_utils import split_sql_table logger = logging.getLogger(__name__) @@ -138,12 +139,14 @@ def _parse_sql_table_with_default_schema( """Split ``sql_table`` into ``(schema, table)``, falling back to ``datasource.schema_name`` when the name is unqualified. This honours attached SQLite schemas instead of silently using ``main``. + + The schema is everything before the FINAL dot, so a hand-written + Snowflake ``db.schema.table`` or BigQuery ``project.dataset.table`` keeps + its catalog rather than losing it to a split on the first dot. """ default_schema = getattr(datasource, "schema_name", None) or None - if "." in sql_table: - schema_name, _, table_name = sql_table.partition(".") - return (schema_name or None), table_name - return default_schema, sql_table + schema_name, table_name = split_sql_table(sql_table) + return (schema_name if schema_name is not None else default_schema), table_name def _safe_probe( @@ -319,10 +322,15 @@ def refine_dict_with_live_schema(d: dict, datasource: DatasourceConfig) -> bool: return False # Local import to avoid circular import at module load time. - from slayer.engine.schema_drift import _live_schema_for_datasource + from slayer.engine.schema_drift import ( + _live_schema_for_datasource, + _resolve_live_table, + ) - live = _live_schema_for_datasource(datasource=datasource) - table = live.get(sql_table) + live = _live_schema_for_datasource( + datasource=datasource, schemas=[split_sql_table(sql_table)[0]], + ) + table = _resolve_live_table(sql_table=sql_table, live_tables=live) if table is None: return False live_columns = table.columns diff --git a/tests/test_ingestion.py b/tests/test_ingestion.py index 591807a5..50fda67d 100644 --- a/tests/test_ingestion.py +++ b/tests/test_ingestion.py @@ -52,7 +52,16 @@ class TestGetColumnsFallback: """Tests for _get_columns_fallback parameterized queries.""" def test_without_schema(self): - engine, conn = _setup_mock_engine([("id", "INTEGER"), ("name", "VARCHAR")]) + """With no schema the query cannot be narrowed, so it selects the + catalog and schema alongside the columns and groups the rows by them. + Unioning every match — which is what selecting only the columns did — + produced models referencing columns their table does not have.""" + engine, conn = _setup_mock_engine( + [ + ("db", "public", "id", "INTEGER"), + ("db", "public", "name", "VARCHAR"), + ] + ) result = _get_columns_fallback(sa_engine=engine, table_name="orders", schema=None) assert len(result) == 2 @@ -65,7 +74,8 @@ def test_without_schema(self): assert isinstance(sql_text, sa.TextClause) sql_str = str(sql_text) assert ":table_name" in sql_str - assert "table_schema" not in sql_str + # No schema was supplied, so none may be bound as a filter. + assert ":schema" not in sql_str params = args[1] if len(args) > 1 else kwargs assert params == {"table_name": "orders"} diff --git a/tests/test_ingestion_schema_qualification.py b/tests/test_ingestion_schema_qualification.py new file mode 100644 index 00000000..f84a5a90 --- /dev/null +++ b/tests/test_ingestion_schema_qualification.py @@ -0,0 +1,1810 @@ +"""Ingested models must keep enough schema information to be queryable. + +A bare ``slayer ingest`` against DuckDB swept *every* schema and wrote each +object's bare name into ``sql_table``, so a model in a non-default schema +generated ``FROM reports`` and failed with a table-not-found error. Models in +the connection's default schema resolved via the search path, which is what +made the breakage look partial rather than systemic. + +Three defects are pinned here: + +* **D1** — cross-schema discovery with no schema recorded. +* **D2** — same-named tables in two schemas silently merged their columns, + because the ``INFORMATION_SCHEMA`` column fallback ran unfiltered. +* **D3** — ``datasources create --schema X --ingest`` discarded ``schema_name`` + while ``validate-models`` read it back. + +The token discipline these tests enforce is easy to get backwards, so it is +worth stating: on DuckDB ``get_schema_names()`` returns **catalog-qualified** +tokens, and the qualified token is the *safe* one. Measured, a bare ``main`` +token makes ``get_table_names`` and ``has_table`` reach into ``ATTACH``ed +catalogs and makes the column fallback return the union across catalogs. So the +discovery token stays qualified end to end, and the column fallback filters on +``table_catalog`` as well as ``table_schema``. Separately — and this is a +different string — the qualifier written into ``sql_table`` is the bare last +segment, because the connection's current catalog is already the right one. +""" +from __future__ import annotations + +import io +import sqlite3 +import sys +from pathlib import Path +from types import SimpleNamespace +from unittest.mock import MagicMock + +import pytest +import sqlalchemy as sa +from fastapi.testclient import TestClient +from pydantic import BaseModel +from sqlalchemy.pool import StaticPool + +duckdb = pytest.importorskip("duckdb") + +from slayer.api.server import create_app +from slayer.cli import _run_datasources_create, _run_ingest, main +from slayer.core.enums import DataType +from slayer.core.models import Column, DatasourceConfig, SlayerModel +from slayer.core.query import SlayerQuery +from slayer.engine import ingestion as ingestion_module +from slayer.engine.ingestion import ( + IngestableObject, + IngestSchemaScope, + ProcessTableOutcome, + ResolvedSchema, + SkippedTable, + _additive_merge_existing, + _assign_model_names, + _bare_table_name, + _is_system_schema, + _print_ingest_addition, + _schema_of, + ingest_datasource, + ingest_datasource_idempotent, + ingest_datasource_report, + list_ingestable_objects, + list_ingestable_objects_multi, + qualify_sql_table, + resolve_ingest_schemas, + split_sql_table, +) +from slayer.engine.introspect_utils import _get_columns_fallback, _safe_get_columns +from slayer.engine.query_engine import SlayerQueryEngine +from slayer.engine.schema_drift import ( + LiveTable, + ModelAddition, + WholeModelDelete, + _live_schema_for_datasource, + _resolve_live_table, + validate_datasource, +) +from slayer.mcp.server import create_mcp_server +from slayer.storage.type_refinement import _parse_sql_table_with_default_schema +from slayer.storage.yaml_storage import YAMLStorage + +# --------------------------------------------------------------------------- +# Fixtures — real DuckDB files, unit-scoped (the pattern already used by +# tests/test_cube_js_e2e_duckdb.py). No @pytest.mark.integration. +# --------------------------------------------------------------------------- + + +def _seed_repro(db_path: str) -> None: + """``main.in_default`` + ``openfda_rest.reports`` — the reported shape.""" + con = duckdb.connect(db_path) + con.execute("CREATE TABLE in_default(x INTEGER)") + con.execute("INSERT INTO in_default VALUES (1)") + con.execute("CREATE SCHEMA openfda_rest") + con.execute("CREATE TABLE openfda_rest.reports(id INTEGER, n INTEGER)") + con.execute("INSERT INTO openfda_rest.reports VALUES (1, 10), (2, 20)") + con.close() + + +def _seed_collide(db_path: str) -> None: + """``main.reports(a)`` + ``s2.reports(b, c)`` — one name, two schemas.""" + con = duckdb.connect(db_path) + con.execute("CREATE TABLE reports(a INTEGER)") + con.execute("CREATE SCHEMA s2") + con.execute("CREATE TABLE s2.reports(b INTEGER, c INTEGER)") + con.close() + + +def _seed_three_schemas(db_path: str) -> None: + """``main.only_main`` + ``s2.reports`` + ``s3.reports`` — collision between + two *non-default* schemas, so the default-wins rule cannot decide it.""" + con = duckdb.connect(db_path) + con.execute("CREATE TABLE only_main(z INTEGER)") + con.execute("CREATE SCHEMA s2") + con.execute("CREATE TABLE s2.reports(b INTEGER)") + con.execute("CREATE SCHEMA s3") + con.execute("CREATE TABLE s3.reports(c INTEGER)") + con.close() + + +def _duckdb_ds(db_path: str, *, name: str = "ds", **kw) -> DatasourceConfig: + return DatasourceConfig(name=name, type="duckdb", database=db_path, **kw) + + +def _repro_ds(tmp_path: Path) -> DatasourceConfig: + db_path = str(tmp_path / "fda.duckdb") + _seed_repro(db_path) + return _duckdb_ds(db_path) + + +def _collide_ds(tmp_path: Path) -> DatasourceConfig: + db_path = str(tmp_path / "collide.duckdb") + _seed_collide(db_path) + return _duckdb_ds(db_path) + + +def _inspector_for(ds: DatasourceConfig) -> tuple[sa.Engine, sa.engine.Inspector]: + eng = sa.create_engine(f"duckdb:///{ds.database}", poolclass=StaticPool) + return eng, sa.inspect(eng) + + +# --- attached-catalog fixture ---------------------------------------------- +# +# The primary file is ``att_main.duckdb`` (so the current catalog is +# ``att_main``) and the attached one is registered as ``aaa`` — deliberately +# sorting BEFORE the default catalog, so a "lowest sorted wins" tie-break would +# pick the wrong one and be caught. + + +def _seed_attached_pair(tmp_path: Path) -> tuple[str, str]: + main_path = str(tmp_path / "att_main.duckdb") + other_path = str(tmp_path / "att_other.duckdb") + + con = duckdb.connect(main_path) + con.execute("CREATE TABLE in_default(x INTEGER)") + con.execute("CREATE TABLE shared(m INTEGER)") + con.execute("CREATE SCHEMA openfda_rest") + con.execute("CREATE TABLE openfda_rest.reports(id INTEGER, n INTEGER)") + con.close() + + con = duckdb.connect(other_path) + con.execute("CREATE TABLE only_in_other(y INTEGER)") + con.execute("CREATE TABLE shared(o INTEGER)") + con.close() + return main_path, other_path + + +def _attached_engine(main_path: str, other_path: str) -> sa.Engine: + """A DuckDB engine on ``main_path`` with ``other_path`` attached as ``aaa``. + + ``StaticPool`` so the single DBAPI connection carrying the ``ATTACH`` + is the one every later ``Inspector`` call reuses. + """ + eng = sa.create_engine(f"duckdb:///{main_path}", poolclass=StaticPool) + with eng.connect() as conn: + conn.exec_driver_sql(f"ATTACH '{other_path}' AS aaa") + conn.commit() + return eng + + +@pytest.fixture +def attached(tmp_path, monkeypatch): + """An attached-catalog datasource whose engine factory always yields a + freshly-attached engine (ingestion disposes the engine it is handed).""" + main_path, other_path = _seed_attached_pair(tmp_path) + ds = _duckdb_ds(main_path) + + def _factory(_ds, *_a, **_kw): + return _attached_engine(main_path, other_path) + + monkeypatch.setattr("slayer.sql.engine_factory.get_engine", _factory) + return SimpleNamespace( + ds=ds, + main_path=main_path, + other_path=other_path, + engine=lambda: _attached_engine(main_path, other_path), + ) + + +def _storage(tmp_path: Path, *, sub: str = "store") -> YAMLStorage: + return YAMLStorage(base_dir=str(tmp_path / sub)) + + +async def _ingest(ds: DatasourceConfig, storage: YAMLStorage, **kw): + await storage.save_datasource(ds) + return await ingest_datasource_idempotent( + datasource=ds, storage=storage, **kw + ) + + +def _by_name(models: list[SlayerModel]) -> dict[str, SlayerModel]: + return {m.name: m for m in models} + + +def _sole_value(resp): + assert resp.row_count == 1, f"expected exactly 1 row: {resp.data}" + return next(iter(resp.data[0].values())) + + +async def _count(engine: SlayerQueryEngine, model_name: str) -> int: + resp = await engine.execute( + SlayerQuery( + source_model=model_name, + measures=[{"formula": "*:count", "name": "cnt"}], + ) + ) + return _sole_value(resp) + + +# --------------------------------------------------------------------------- +# 1-3. Regression / end-to-end +# --------------------------------------------------------------------------- + + +class TestReportedRegression: + async def test_bare_ingest_covers_only_the_default_schema(self, tmp_path): + """Test 1. Bare ingest must resolve to the connection's default schema + only — DuckDB is the one Tier-1 dialect whose ``schema=None`` sweeps + every schema, which is how ``openfda_rest.reports`` got a bare + ``sql_table``.""" + ds = _repro_ds(tmp_path) + models = ingest_datasource(datasource=ds) + + assert _by_name(models).keys() == {"in_default"} + assert _by_name(models)["in_default"].sql_table == "in_default" + + async def test_bare_ingest_reports_the_other_schemas(self, tmp_path): + """Test 1 (cont). Narrowing the scan must not silently lose models — + the user is told which schemas were left out and how to get them.""" + ds = _repro_ds(tmp_path) + eng, insp = _inspector_for(ds) + try: + scope = resolve_ingest_schemas( + inspector=insp, + requested=None, + all_schemas=False, + datasource_schema=None, + ) + finally: + eng.dispose() + + assert [s.name.rsplit(".", 1)[-1] for s in scope.schemas] == ["main"] + assert scope.schemas[0].is_default is True + assert scope.schemas[0].explicit is False + assert scope.other_schemas == ["openfda_rest"] + + async def test_explicit_schema_qualifies_and_is_queryable(self, tmp_path): + """Test 2. The issue's exact repro: ingest the non-default schema and + run ``*:count`` against it. Before the fix the generated SQL is + ``FROM reports`` and DuckDB raises a catalog error.""" + ds = _repro_ds(tmp_path) + storage = _storage(tmp_path) + result = await _ingest(ds, storage, schemas=["openfda_rest"]) + assert not result.errors, result.errors + + model = await storage.get_model("reports", data_source="ds") + assert model.sql_table == "openfda_rest.reports" + assert await _count(SlayerQueryEngine(storage=storage), "reports") == 2 + + async def test_all_schemas_qualifies_only_non_default(self, tmp_path): + """Test 3. Default-schema objects stay unqualified so turning the flag + on never rewrites models that already exist; everything else is + qualified. Both must be queryable.""" + ds = _repro_ds(tmp_path) + storage = _storage(tmp_path) + result = await _ingest(ds, storage, all_schemas=True) + assert not result.errors, result.errors + + in_default = await storage.get_model("in_default", data_source="ds") + reports = await storage.get_model("reports", data_source="ds") + assert in_default.sql_table == "in_default" + assert reports.sql_table == "openfda_rest.reports" + + engine = SlayerQueryEngine(storage=storage) + assert await _count(engine, "in_default") == 1 + assert await _count(engine, "reports") == 2 + + +# --------------------------------------------------------------------------- +# 4-6. D2 — column corruption +# --------------------------------------------------------------------------- + + +class TestColumnCorruption: + async def test_bare_ingest_does_not_union_columns_across_schemas( + self, tmp_path + ): + """Test 4. ``main.reports(a)`` and ``s2.reports(b, c)`` are different + tables. The schema-blind ``INFORMATION_SCHEMA`` query returned all + three columns, producing a model that references columns its table + does not have.""" + ds = _collide_ds(tmp_path) + models = ingest_datasource(datasource=ds) + + reports = _by_name(models)["reports"] + assert [c.name for c in reports.columns] == ["a"] + + def test_safe_get_columns_resolves_none_to_the_default_schema( + self, tmp_path + ): + """Test 5. ``_safe_get_columns`` holds the Inspector, so it can resolve + ``None`` before the fallback ever runs unfiltered.""" + ds = _collide_ds(tmp_path) + eng, insp = _inspector_for(ds) + try: + cols = _safe_get_columns(insp, eng, "reports", None) + finally: + eng.dispose() + + assert [c["name"] for c in cols] == ["a"] + + def test_discovery_reports_one_entry_carrying_its_schema(self, tmp_path): + """Test 6. Discovery must return one object tagged with the schema it + was found in, not two indistinguishable bare duplicates.""" + ds = _collide_ds(tmp_path) + eng, insp = _inspector_for(ds) + try: + objects = list_ingestable_objects(inspector=insp, schema=None) + finally: + eng.dispose() + + assert [o.name for o in objects] == ["reports"] + assert objects[0].schema.rsplit(".", 1)[-1] == "main" + + +# --------------------------------------------------------------------------- +# 7-9. Collisions +# --------------------------------------------------------------------------- + + +class TestCollisions: + async def test_all_schemas_collision_gives_the_default_schema_the_name( + self, tmp_path + ): + """Test 7. Two schemas claim model name ``reports``. The default + schema wins; the loser is skipped, never suffixed — suffixes shift + with the object set and orphan models.""" + ds = _collide_ds(tmp_path) + report = ingest_datasource_report(datasource=ds, all_schemas=True) + + reports = _by_name(report.models)["reports"] + assert reports.sql_table == "reports" + assert [c.name for c in reports.columns] == ["a"] + + losers = [s for s in report.skipped if "s2" in s.table_name] + assert len(losers) == 1 + assert "collision" in losers[0].reason + + async def test_non_default_collision_is_order_independent(self, tmp_path): + """Test 8. Neither schema is the default, so the winner is fixed by + the schema name itself — not by argument order or listing order.""" + db_path = str(tmp_path / "three.duckdb") + _seed_three_schemas(db_path) + ds = _duckdb_ds(db_path) + + forward = ingest_datasource_report(datasource=ds, schemas=["s2", "s3"]) + reverse = ingest_datasource_report(datasource=ds, schemas=["s3", "s2"]) + + for report in (forward, reverse): + assert _by_name(report.models)["reports"].sql_table == "s2.reports" + assert {s.table_name for s in forward.skipped} == { + s.table_name for s in reverse.skipped + } + + def test_single_schema_sanitization_behaviour_is_unchanged(self, tmp_path): + """Test 9. With one schema in scope the cross-schema tie-break keys + never fire, so ``__`` sanitization stays byte-identical. The real + guard is tests/test_ingestion_name_sanitize.py passing unchanged; + this pins the same rule at the helper.""" + objects = [ + IngestableObject(name="a_b", kind="table", schema="main"), + IngestableObject(name="a__b", kind="table", schema="main"), + ] + assigned, skipped = _assign_model_names(objects) + + assert assigned[("main", "a_b")] == "a_b" + assert ("main", "a__b") not in assigned + assert [s.table_name for s in skipped] == ["a__b"] + + +# --------------------------------------------------------------------------- +# 10-12. Self-heal +# --------------------------------------------------------------------------- + + +def _persisted_reports_model(*, sql_table: str, columns: list[str]) -> SlayerModel: + """A model carrying hand-authored metadata the additive contract must + preserve verbatim through a qualifier repair.""" + return SlayerModel( + name="reports", + data_source="ds", + sql_table=sql_table, + description="hand written", + columns=[ + Column( + name=name, + type=DataType.INT, + description=f"desc for {name}", + label=f"Label {name}", + ) + for name in columns + ], + measures=[{"name": "total_n", "formula": "n:sum"}], + ) + + +class TestSelfHeal: + async def test_missing_qualifier_is_healed_and_metadata_preserved( + self, tmp_path + ): + """Test 10. Re-ingest repairs a *missing* qualifier. It usually changes + no columns and no joins, so the repair has to participate in the + short-circuit or the merged model is computed and discarded.""" + ds = _repro_ds(tmp_path) + storage = _storage(tmp_path) + await storage.save_datasource(ds) + await storage.save_model( + _persisted_reports_model(sql_table="reports", columns=["id", "n"]) + ) + + result = await _ingest(ds, storage, schemas=["openfda_rest"]) + + saved = await storage.get_model("reports", data_source="ds") + assert saved.sql_table == "openfda_rest.reports" + assert saved.description == "hand written" + assert [c.description for c in saved.columns] == [ + "desc for id", + "desc for n", + ] + assert [c.label for c in saved.columns] == ["Label id", "Label n"] + assert [m.name for m in saved.measures] == ["total_n"] + + addition = next(a for a in result.additions if a.model_name == "reports") + assert addition.sql_table_change == "reports → openfda_rest.reports" + + async def test_existing_qualifier_is_never_rewritten(self, tmp_path): + """Test 11. Healing adds a qualifier; it never replaces one. A model + deliberately pointed at ``prod.reports`` stays there.""" + ds = _repro_ds(tmp_path) + storage = _storage(tmp_path) + await storage.save_datasource(ds) + await storage.save_model( + _persisted_reports_model(sql_table="prod.reports", columns=["id"]) + ) + + await _ingest(ds, storage, schemas=["openfda_rest"]) + + saved = await storage.get_model("reports", data_source="ds") + assert saved.sql_table == "prod.reports" + + async def test_default_schema_model_is_untouched(self, tmp_path): + """Test 12. A default-schema model is persisted unqualified by design, + and re-ingest must not churn it into ``main.in_default``.""" + ds = _repro_ds(tmp_path) + storage = _storage(tmp_path) + await storage.save_datasource(ds) + await storage.save_model( + SlayerModel( + name="in_default", + data_source="ds", + sql_table="in_default", + columns=[Column(name="x", type=DataType.INT)], + ) + ) + + result = await _ingest(ds, storage) + + saved = await storage.get_model("in_default", data_source="ds") + assert saved.sql_table == "in_default" + for addition in result.additions: + if addition.model_name == "in_default": + assert addition.sql_table_change is None + assert addition.new_columns == [] + + +# --------------------------------------------------------------------------- +# 13, 27. Cross-schema merge guard +# --------------------------------------------------------------------------- + + +class TestCrossSchemaMergeGuard: + async def test_sequential_single_schema_ingests_do_not_fuse(self, tmp_path): + """Test 13. Two explicit single-schema ingests must not merge two + different physical tables into one model. No new flag is involved — + this fuses today.""" + ds = _collide_ds(tmp_path) + storage = _storage(tmp_path) + await _ingest(ds, storage, schemas=["main"]) + result = await _ingest(ds, storage, schemas=["s2"]) + + saved = await storage.get_model("reports", data_source="ds") + assert [c.name for c in saved.columns] == ["a"] + assert saved.sql_table == "main.reports" + assert any("cross-schema" in s.reason for s in result.skipped), ( + result.skipped + ) + + async def test_bare_persisted_model_is_not_repointed(self, tmp_path): + """Test 27. The hole that qualifying only non-default schemas opens: a + default-schema model is persisted unqualified, so a schema comparison + alone cannot fire. Without the live ``has_table`` probe the self-heal + happily repoints ``reports`` at ``s2.reports``.""" + ds = _collide_ds(tmp_path) + storage = _storage(tmp_path) + await _ingest(ds, storage) # bare -> sql_table: reports, columns [a] + + before = await storage.get_model("reports", data_source="ds") + assert before.sql_table == "reports" + + result = await _ingest(ds, storage, schemas=["s2"]) + + saved = await storage.get_model("reports", data_source="ds") + assert saved.sql_table == "reports" + assert [c.name for c in saved.columns] == ["a"] + assert any("cross-schema" in s.reason for s in result.skipped), ( + result.skipped + ) + + +# --------------------------------------------------------------------------- +# 14-16. validate-models +# --------------------------------------------------------------------------- + + +class TestValidateModels: + async def test_multi_schema_models_are_not_marked_for_deletion( + self, tmp_path + ): + """Test 14. The data-loss guard. A qualified model that the live map + cannot resolve becomes a ``WholeModelDelete``, which + ``validate-models --force-clean`` acts on.""" + ds = _repro_ds(tmp_path) + storage = _storage(tmp_path) + await _ingest(ds, storage, all_schemas=True) + + models = [ + await storage.get_model(n, data_source="ds") + for n in ("in_default", "reports") + ] + to_delete = await validate_datasource(datasource=ds, models=models) + + whole = [e for e in to_delete if isinstance(e, WholeModelDelete)] + assert whole == [], whole + + async def test_qualified_model_diffs_against_its_own_table(self, tmp_path): + """Test 15. With two same-named tables, resolving via the bare-name + fallback can pick the wrong live table and report phantom drift.""" + ds = _collide_ds(tmp_path) + model = SlayerModel( + name="reports_s2", + data_source="ds", + sql_table="s2.reports", + columns=[ + Column(name="b", type=DataType.INT), + Column(name="c", type=DataType.INT), + ], + ) + to_delete = await validate_datasource(datasource=ds, models=[model]) + assert to_delete == [], to_delete + + async def test_view_in_a_non_default_schema_still_resolves(self, tmp_path): + """Test 16. Views are included in the live map unconditionally; the + schema-awareness change must not quietly re-arm the view blindness + that made view-backed models look deleted.""" + db_path = str(tmp_path / "views.duckdb") + con = duckdb.connect(db_path) + con.execute("CREATE SCHEMA analytics") + con.execute("CREATE TABLE analytics.orders(id INTEGER, amount INTEGER)") + con.execute( + "CREATE VIEW analytics.stg_orders AS SELECT id, amount " + "FROM analytics.orders" + ) + con.close() + ds = _duckdb_ds(db_path) + + model = SlayerModel( + name="stg_orders", + data_source="ds", + sql_table="analytics.stg_orders", + columns=[ + Column(name="id", type=DataType.INT), + Column(name="amount", type=DataType.INT), + ], + ) + to_delete = await validate_datasource(datasource=ds, models=[model]) + assert [e for e in to_delete if isinstance(e, WholeModelDelete)] == [] + + +# --------------------------------------------------------------------------- +# 17-19, 33. Parser hardening (dialect-free) +# --------------------------------------------------------------------------- + + +class TestDottedNameParsers: + @pytest.mark.parametrize( + "sql_table,expected", + [("t", "t"), ("s.t", "t"), ("c.s.t", "t")], + ) + def test_bare_table_name_takes_the_last_segment(self, sql_table, expected): + """Test 17. ``split(".", 1)[1]`` returns ``s.t`` for a three-part name, + so every in-scope-table comparison built on it missed.""" + assert _bare_table_name(sql_table) == expected + + @pytest.mark.parametrize( + "sql_table,expected", + [ + ("t", (None, "t")), + ("s.t", ("s", "t")), + ("c.s.t", ("c.s", "t")), + ("proj.dataset.tbl", ("proj.dataset", "tbl")), + ], + ) + def test_split_sql_table_preserves_the_catalog(self, sql_table, expected): + """Tests 18/33. The schema token is everything before the final dot. + Truncating to the last two segments would discard catalog identity — + which on DuckDB matches nothing at all.""" + assert split_sql_table(sql_table) == expected + + def test_parse_with_default_schema_preserves_the_catalog(self): + """Test 33. Snowflake ``db.schema.table`` and BigQuery + ``project.dataset.table`` are hand-writable today, and ``partition`` + split them after the FIRST dot.""" + ds = _duckdb_ds(":memory:", schema_name="fallback") + assert _parse_sql_table_with_default_schema( + "proj.dataset.tbl", ds + ) == ("proj.dataset", "tbl") + assert _parse_sql_table_with_default_schema("s.t", ds) == ("s", "t") + assert _parse_sql_table_with_default_schema("t", ds) == ("fallback", "t") + + def test_schema_of_returns_the_bare_segment(self): + assert _schema_of("t") is None + assert _schema_of("s.t") == "s" + assert _schema_of("c.s.t") == "s" + + def test_resolve_live_table_walks_progressively_shorter_keys(self): + """Test 19. A three-part ``sql_table`` must resolve against a live map + keyed either way — and must return None rather than a wrong match when + the short key is ambiguous.""" + live = LiveTable(columns={"a": DataType.INT}) + assert _resolve_live_table( + sql_table="c.s.t", live_tables={"s.t": live} + ) is live + assert _resolve_live_table( + sql_table="c.s.t", live_tables={"t": live} + ) is live + assert _resolve_live_table( + sql_table="c.s.t", live_tables={"other.t": live} + ) is None + + def test_resolve_live_table_still_unquotes_identifiers(self): + """The existing quoted-identifier path must survive the candidate-list + rewrite (``prod."Company"`` for case-sensitive Postgres).""" + live = LiveTable(columns={"a": DataType.INT}) + assert _resolve_live_table( + sql_table='prod."Company"', live_tables={"Company": live} + ) is live + + +# --------------------------------------------------------------------------- +# 20-22. schema_name persistence +# --------------------------------------------------------------------------- + + +def _create_args(tmp_path: Path, db_path: str, **overrides) -> SimpleNamespace: + base = dict( + connection_string=f"duckdb:///{db_path}", + name="ds", + description=None, + ingest=True, + include=None, + exclude=None, + schema=None, + all_schemas=False, + include_views=True, + yes=True, + storage=str(tmp_path / "store"), + models_dir=None, + ) + base.update(overrides) + return SimpleNamespace(**base) + + +class TestSchemaNamePersistence: + async def test_create_with_schema_persists_it_and_bare_ingest_reuses_it( + self, tmp_path + ): + """Test 20. ``--schema`` was used for the one-shot ingest and thrown + away, while ``validate-models`` read ``schema_name`` back — so the two + commands looked at different schemas. Drives the real CLI entry + points, which is also what pins the persist-then-ingest call order.""" + db_path = str(tmp_path / "fda.duckdb") + _seed_repro(db_path) + storage = _storage(tmp_path) + + _run_datasources_create( + _create_args(tmp_path, db_path, schema="openfda_rest"), storage + ) + + ds = await storage.get_datasource("ds") + assert ds.schema_name == "openfda_rest" + + _run_ingest( + SimpleNamespace( + datasource="ds", + schema=None, + all_schemas=False, + include=None, + exclude=None, + include_views=True, + storage=str(tmp_path / "store"), + models_dir=None, + ) + ) + model = await storage.get_model("reports", data_source="ds") + assert model.sql_table == "openfda_rest.reports" + + async def test_multi_schema_create_does_not_persist_schema_name( + self, tmp_path + ): + """Test 21. ``schema_name`` is a single-schema default. A CSV list or + ``--all-schemas`` has no single value to persist, and persisting the + first would silently narrow every later bare ingest.""" + db_path = str(tmp_path / "fda.duckdb") + _seed_repro(db_path) + + storage = _storage(tmp_path, sub="csv") + _run_datasources_create( + _create_args( + tmp_path, db_path, schema="main,openfda_rest", + storage=str(tmp_path / "csv"), + ), + storage, + ) + assert (await storage.get_datasource("ds")).schema_name is None + + storage2 = _storage(tmp_path, sub="all") + _run_datasources_create( + _create_args( + tmp_path, db_path, all_schemas=True, + storage=str(tmp_path / "all"), + ), + storage2, + ) + assert (await storage2.get_datasource("ds")).schema_name is None + + async def test_precedence_explicit_beats_persisted_beats_default( + self, tmp_path + ): + """Test 22. ``datasource.schema_name`` is a fallback consulted only + when nothing more specific was given — never a conflict.""" + db_path = str(tmp_path / "fda.duckdb") + _seed_repro(db_path) + ds = _duckdb_ds(db_path, schema_name="openfda_rest") + + persisted = ingest_datasource_report(datasource=ds) + assert _by_name(persisted.models).keys() == {"reports"} + + explicit = ingest_datasource_report(datasource=ds, schemas=["main"]) + assert _by_name(explicit.models).keys() == {"in_default"} + + plain = ingest_datasource_report(datasource=_duckdb_ds(db_path)) + assert _by_name(plain.models).keys() == {"in_default"} + + +# --------------------------------------------------------------------------- +# 23-25, 34. Surface parity and the conflict matrix +# --------------------------------------------------------------------------- + + +class TestEngineConflicts: + @pytest.mark.parametrize( + "kwargs", + [ + {"schema": "a", "schemas": ["b"]}, + {"all_schemas": True, "schema": "a"}, + {"all_schemas": True, "schemas": ["b"]}, + ], + ) + def test_report_rejects_conflicting_scope_arguments(self, tmp_path, kwargs): + """Tests 25/34. One shared rule, enforced at every entry point, so the + CLI's mutually-exclusive group is not the only thing holding the line.""" + ds = _repro_ds(tmp_path) + with pytest.raises(ValueError): + ingest_datasource_report(datasource=ds, **kwargs) + + @pytest.mark.parametrize( + "kwargs", + [ + {"schema": "a", "schemas": ["b"]}, + {"all_schemas": True, "schema": "a"}, + {"all_schemas": True, "schemas": ["b"]}, + ], + ) + def test_ingest_datasource_rejects_conflicting_scope_arguments( + self, tmp_path, kwargs + ): + ds = _repro_ds(tmp_path) + with pytest.raises(ValueError): + ingest_datasource(datasource=ds, **kwargs) + + @pytest.mark.parametrize( + "kwargs", + [ + {"schema": "a", "schemas": ["b"]}, + {"all_schemas": True, "schema": "a"}, + {"all_schemas": True, "schemas": ["b"]}, + ], + ) + async def test_idempotent_rejects_conflicting_scope_arguments( + self, tmp_path, kwargs + ): + ds = _repro_ds(tmp_path) + storage = _storage(tmp_path) + await storage.save_datasource(ds) + with pytest.raises(ValueError): + await ingest_datasource_idempotent( + datasource=ds, storage=storage, **kwargs + ) + + +class TestRestParity: + def _client(self, tmp_path, ds: DatasourceConfig): + storage = _storage(tmp_path) + client = TestClient(create_app(storage=storage)) + resp = client.post( + "/datasources", + json={"name": ds.name, "type": ds.type, "database": ds.database}, + ) + assert resp.status_code < 300, resp.text + return client, storage + + async def test_schemas_list_is_honoured(self, tmp_path): + """Test 23.""" + ds = _repro_ds(tmp_path) + client, storage = self._client(tmp_path, ds) + resp = client.post( + "/ingest", json={"datasource": "ds", "schemas": ["openfda_rest"]} + ) + assert resp.status_code == 200, resp.text + model = await storage.get_model("reports", data_source="ds") + assert model.sql_table == "openfda_rest.reports" + + async def test_all_schemas_is_honoured(self, tmp_path): + """Test 23 (cont).""" + ds = _repro_ds(tmp_path) + client, storage = self._client(tmp_path, ds) + resp = client.post( + "/ingest", json={"datasource": "ds", "all_schemas": True} + ) + assert resp.status_code == 200, resp.text + assert ( + await storage.get_model("reports", data_source="ds") + ).sql_table == "openfda_rest.reports" + assert ( + await storage.get_model("in_default", data_source="ds") + ).sql_table == "in_default" + + @pytest.mark.parametrize( + "body", + [ + {"schema_name": "a", "schemas": ["b"]}, + {"all_schemas": True, "schema_name": "a"}, + {"all_schemas": True, "schemas": ["b"]}, + ], + ) + def test_conflicting_scope_arguments_are_422(self, tmp_path, body): + """Tests 23/34. A conflict is a client error, not a silent preference + for whichever argument the handler happens to read first.""" + ds = _repro_ds(tmp_path) + client, _ = self._client(tmp_path, ds) + resp = client.post("/ingest", json={"datasource": "ds", **body}) + assert resp.status_code == 422, resp.text + + +class TestMcpParity: + async def _call(self, storage, **kwargs) -> str: + server = create_mcp_server(storage=storage) + content, _ = await server.call_tool( + name="ingest_datasource_models", + arguments={"datasource_name": "ds", **kwargs}, + ) + return content[0].text + + async def test_schemas_csv_is_honoured(self, tmp_path): + """Test 24. Comma-separated to match ``include_tables``' existing + style rather than introducing a second list convention.""" + ds = _repro_ds(tmp_path) + storage = _storage(tmp_path) + await storage.save_datasource(ds) + + await self._call(storage, schemas="main,openfda_rest") + assert ( + await storage.get_model("reports", data_source="ds") + ).sql_table == "openfda_rest.reports" + assert ( + await storage.get_model("in_default", data_source="ds") + ).sql_table == "in_default" + + async def test_all_schemas_is_honoured(self, tmp_path): + """Test 24 (cont).""" + ds = _repro_ds(tmp_path) + storage = _storage(tmp_path) + await storage.save_datasource(ds) + + await self._call(storage, all_schemas=True) + assert ( + await storage.get_model("reports", data_source="ds") + ).sql_table == "openfda_rest.reports" + + @pytest.mark.parametrize( + "kwargs", + [ + {"schema_name": "a", "schemas": "b"}, + {"all_schemas": True, "schema_name": "a"}, + {"all_schemas": True, "schemas": "b"}, + ], + ) + async def test_conflicting_scope_arguments_are_reported( + self, tmp_path, kwargs + ): + """Test 34. MCP returns an error string rather than raising — the + agent has to be able to read and correct it.""" + ds = _repro_ds(tmp_path) + storage = _storage(tmp_path) + await storage.save_datasource(ds) + + out = await self._call(storage, **kwargs) + assert "schema" in out.lower() + assert any(w in out.lower() for w in ("cannot", "conflict", "both")) + + +# --------------------------------------------------------------------------- +# 26. Non-regression for single-schema dialects +# --------------------------------------------------------------------------- + + +class TestUnqualifiedNonRegression: + def test_sqlite_ingest_stays_unqualified(self, tmp_path): + """Test 26. SQLite reports ``main`` as its default schema. Qualifying + it would rewrite every existing model on disk for no benefit, and the + SQLite fixtures across the suite pin the unqualified form.""" + db_path = str(tmp_path / "live.db") + conn = sqlite3.connect(db_path) + conn.executescript( + "CREATE TABLE orders (id INTEGER PRIMARY KEY, amount REAL);" + "CREATE VIEW v_orders AS SELECT id FROM orders;" + ) + conn.commit() + conn.close() + + ds = DatasourceConfig(name="ds", type="sqlite", database=db_path) + models = _by_name(ingest_datasource(datasource=ds)) + + assert models["orders"].sql_table == "orders" + assert models["v_orders"].sql_table == "v_orders" + + def test_duckdb_default_schema_stays_unqualified(self, tmp_path): + """Test 26 (cont). Same rule on the dialect that exposed the bug.""" + ds = _repro_ds(tmp_path) + models = _by_name(ingest_datasource(datasource=ds, schemas=None)) + assert models["in_default"].sql_table == "in_default" + + +# --------------------------------------------------------------------------- +# 28-30a, 32. Attached catalogs and the column fallback +# --------------------------------------------------------------------------- + + +class TestAttachedCatalogs: + def test_all_schemas_covers_only_the_current_catalog(self, attached): + """Test 28. ``--all-schemas`` means "this database", not "and whatever + anyone attached to the session". The dropped schemas are reported, not + silently discarded.""" + report = ingest_datasource_report( + datasource=attached.ds, all_schemas=True + ) + + names = _by_name(report.models) + assert "only_in_other" not in names + assert {"in_default", "shared", "reports"} <= set(names) + assert names["reports"].sql_table == "openfda_rest.reports" + assert names["in_default"].sql_table == "in_default" + + dropped = [s for s in report.skipped if "attached catalog" in s.reason] + assert dropped, report.skipped + # The reason has to be actionable: it must name the catalog AND the + # exact invocation that would ingest it. + reason = next(s.reason for s in dropped if "aaa" in s.reason) + assert "aaa.main" in reason + assert "--schema" in reason + + def test_bare_ingest_does_not_reach_into_an_attached_catalog( + self, attached + ): + """Test 29. ``get_table_names(schema="main")`` still returns + ``only_in_other`` from the attached catalog — narrowing to the bare + default schema is not enough, the token must carry the catalog.""" + report = ingest_datasource_report(datasource=attached.ds) + + names = _by_name(report.models) + assert "only_in_other" not in names + assert names["shared"].sql_table == "shared" + assert [c.name for c in names["shared"].columns] == ["m"] + + def test_column_fallback_accepts_a_qualified_token(self, attached): + """Test 30. The measured zero-column trap: filtering on + ``table_schema`` alone means a qualified token matches nothing and the + model is created silently empty.""" + eng = attached.engine() + try: + cols = _get_columns_fallback(eng, "reports", "att_main.openfda_rest") + finally: + eng.dispose() + assert [c["name"] for c in cols] == ["id", "n"] + + def test_column_fallback_never_unions_across_catalogs(self, attached): + """Test 30a. ``shared`` exists in both catalogs under schema ``main``. + Normalising the token to its bare last segment — which an earlier draft + of this work proposed — returns ``['m', 'o']``. This is the test that + keeps that rule from coming back.""" + eng = attached.engine() + try: + qualified = _get_columns_fallback(eng, "shared", "att_main.main") + bare = _get_columns_fallback(eng, "shared", "main") + finally: + eng.dispose() + + assert [c["name"] for c in qualified] == ["m"] + # Pins the hazard itself, so the reason for the qualified token is + # visible in the test rather than only in the commit message. Sorted: + # the union spans two catalogs and ``ORDER BY ordinal_position`` says + # nothing about which catalog's rows come first. + assert sorted(c["name"] for c in bare) == ["m", "o"] + + def test_column_fallback_prefers_the_default_over_the_lowest_sorted( + self, attached + ): + """Test 32. The attached catalog is named ``aaa`` so it sorts first. + A lowest-sorted tie-break would swap union corruption for deterministic + wrong-table corruption, which is harder to notice.""" + eng = attached.engine() + try: + cols = _get_columns_fallback( + eng, "shared", None, default_schema="att_main.main" + ) + finally: + eng.dispose() + assert [c["name"] for c in cols] == ["m"] + + def test_column_fallback_raises_when_it_cannot_disambiguate(self, attached): + """Test 32 (cont). With no default to fall back on, refuse rather than + pick. Per-object isolation turns this into a reported skip, so one + ambiguous object never aborts the run.""" + eng = attached.engine() + try: + with pytest.raises(ValueError) as excinfo: + _get_columns_fallback(eng, "shared", None) + finally: + eng.dispose() + message = str(excinfo.value) + assert "aaa.main" in message and "att_main.main" in message + + def test_all_schemas_never_produces_a_columnless_model(self, attached): + """Test 31. The failure mode this whole token discipline exists to + prevent is a model that persists with zero columns and no error.""" + report = ingest_datasource_report( + datasource=attached.ds, all_schemas=True + ) + assert report.models + for model in report.models: + assert model.columns, f"{model.name} has no columns" + + +# --------------------------------------------------------------------------- +# 35-37. Remaining review-driven cases +# --------------------------------------------------------------------------- + + +class TestCollisionWithSanitization: + @pytest.mark.parametrize("reverse", [False, True]) + def test_exact_name_beats_sanitized_regardless_of_schema_order( + self, reverse + ): + """Test 35. ``s1.a__b`` sanitizes to ``a_b`` and collides with a real + ``s2.a_b``. Resolving in one phase over final model names keeps the + "no sanitization beats sanitization" rule ahead of the schema + tie-break, and keeps the outcome independent of listing order.""" + objects = [ + IngestableObject(name="a__b", kind="table", schema="s1"), + IngestableObject(name="a_b", kind="table", schema="s2"), + ] + if reverse: + objects.reverse() + + assigned, skipped = _assign_model_names(objects) + + assert assigned[("s2", "a_b")] == "a_b" + assert ("s1", "a__b") not in assigned + assert [s.table_name for s in skipped] == ["s1.a__b"] + + +class TestLiveSchemaKeying: + def test_ambiguous_short_keys_are_dropped_not_overwritten(self, attached): + """Test 36. Keying the live map on ``schema.table`` alone lets one + catalog's entry overwrite another's. Full keys are always present; + shorter aliases only when unambiguous.""" + live = _live_schema_for_datasource( + datasource=attached.ds, + schemas=["att_main.main", "aaa.main"], + ) + + assert live["att_main.main.shared"].columns.keys() == {"m"} + assert live["aaa.main.shared"].columns.keys() == {"o"} + # ``main.shared`` and ``shared`` are claimed by both, so neither alias + # may resolve to an arbitrary winner. + assert _resolve_live_table( + sql_table="main.shared", live_tables=live + ) is None + assert _resolve_live_table(sql_table="shared", live_tables=live) is None + + def test_unambiguous_aliases_are_still_inserted(self, attached): + """The alias keys are what let an unqualified legacy model keep + resolving — dropping them wholesale would re-arm the data-loss path.""" + live = _live_schema_for_datasource( + datasource=attached.ds, schemas=["att_main.openfda_rest"], + ) + assert _resolve_live_table( + sql_table="reports", live_tables=live + ) is not None + assert _resolve_live_table( + sql_table="openfda_rest.reports", live_tables=live + ) is not None + + +class TestHint: + def test_hint_fires_for_a_persisted_schema_name(self, tmp_path): + """Test 37. Hint eligibility is "one schema in scope and others exist", + independent of whether that schema was named explicitly. A user who set + ``schema_name`` months ago still needs to hear that a schema appeared.""" + db_path = str(tmp_path / "fda.duckdb") + _seed_repro(db_path) + ds = _duckdb_ds(db_path, schema_name="openfda_rest") + + report = ingest_datasource_report(datasource=ds) + assert report.schema_hint + assert "main" in report.schema_hint + assert "--all-schemas" in report.schema_hint + + async def test_cli_prints_the_hint_and_still_exits_zero( + self, tmp_path, capsys + ): + """Test 1 (cont). Narrowing the default scan is a behaviour change for + DuckDB users, so it has to be visible — but a hint is not a failure.""" + ds = _repro_ds(tmp_path) + storage = _storage(tmp_path) + await storage.save_datasource(ds) + + _run_ingest( + SimpleNamespace( + datasource="ds", + schema=None, + all_schemas=False, + include=None, + exclude=None, + include_views=True, + storage=str(tmp_path / "store"), + models_dir=None, + ) + ) + out = capsys.readouterr().out + assert "openfda_rest" in out + assert "--all-schemas" in out + + +# --------------------------------------------------------------------------- +# Scope resolution unit coverage +# --------------------------------------------------------------------------- + + +class TestScopeResolution: + def test_explicit_schema_is_qualified_verbatim(self): + """An explicitly-named schema is written exactly as given, so + ``--schema public`` keeps producing ``public.orders`` as it does + today — even though ``public`` is Postgres' default.""" + obj = IngestableObject(name="orders", kind="table", schema="public") + resolved = ResolvedSchema(name="public", explicit=True, is_default=True) + assert qualify_sql_table(obj=obj, resolved=resolved) == "public.orders" + + def test_auto_default_schema_is_not_qualified(self): + obj = IngestableObject(name="orders", kind="table", schema="fda.main") + resolved = ResolvedSchema( + name="fda.main", explicit=False, is_default=True + ) + assert qualify_sql_table(obj=obj, resolved=resolved) == "orders" + + def test_auto_non_default_schema_drops_the_catalog(self): + """The emitted qualifier is the bare last segment: the connection's + current catalog is already the right one, so re-stating it would only + break if the datasource is later repointed.""" + obj = IngestableObject(name="reports", kind="table", schema="fda.ofr") + resolved = ResolvedSchema(name="fda.ofr", explicit=False, is_default=False) + assert qualify_sql_table(obj=obj, resolved=resolved) == "ofr.reports" + + def test_all_schemas_excludes_system_schemas(self, tmp_path): + ds = _repro_ds(tmp_path) + eng, insp = _inspector_for(ds) + try: + scope = resolve_ingest_schemas( + inspector=insp, + requested=None, + all_schemas=True, + datasource_schema=None, + ) + finally: + eng.dispose() + + bare = {s.name.rsplit(".", 1)[-1] for s in scope.schemas} + assert bare == {"main", "openfda_rest"} + assert not any( + s.name.startswith(("system.", "temp.")) for s in scope.schemas + ) + + def test_requested_schemas_are_marked_explicit(self, tmp_path): + ds = _repro_ds(tmp_path) + eng, insp = _inspector_for(ds) + try: + scope = resolve_ingest_schemas( + inspector=insp, + requested=["openfda_rest"], + all_schemas=False, + datasource_schema="ignored", + ) + finally: + eng.dispose() + + assert [(s.name, s.explicit) for s in scope.schemas] == [ + ("openfda_rest", True) + ] + + def test_multi_schema_scope_reports_no_hint(self, tmp_path): + """The hint is for "you may be missing something". With more than one + schema in scope there is nothing to nudge about.""" + ds = _repro_ds(tmp_path) + eng, insp = _inspector_for(ds) + try: + scope = resolve_ingest_schemas( + inspector=insp, + requested=["main", "openfda_rest"], + all_schemas=False, + datasource_schema=None, + ) + finally: + eng.dispose() + assert scope.other_schemas == [] + + def test_multi_returns_objects_tagged_with_their_schema(self, tmp_path): + ds = _repro_ds(tmp_path) + eng, insp = _inspector_for(ds) + try: + scope = resolve_ingest_schemas( + inspector=insp, + requested=["main", "openfda_rest"], + all_schemas=False, + datasource_schema=None, + ) + objects = list_ingestable_objects_multi(inspector=insp, scope=scope) + finally: + eng.dispose() + + assert {(o.name, o.schema) for o in objects} == { + ("in_default", "main"), + ("reports", "openfda_rest"), + } + + def test_scope_is_a_pydantic_model(self): + """No dataclasses anywhere in this codebase.""" + assert issubclass(IngestSchemaScope, BaseModel) + assert issubclass(ResolvedSchema, BaseModel) + + +class TestProcessTableOutcome: + def test_outcome_carries_addition_and_skip_separately(self): + """``_process_one_table`` has to be able to say "I declined this one" + as well as "here is what I did" — a skip is not an error and must not + travel as one.""" + outcome = ProcessTableOutcome( + skipped=SkippedTable(table_name="s2.reports", reason="cross-schema") + ) + assert outcome.addition is None + assert outcome.skipped.table_name == "s2.reports" + + +# --------------------------------------------------------------------------- +# Coverage added after the test-plan review +# --------------------------------------------------------------------------- + + +def _seed_fk(db_path: str) -> None: + """A parent/child FK pair in a NON-default schema, so the FK graph has to + be built with the same schema token discovery used.""" + con = duckdb.connect(db_path) + con.execute("CREATE SCHEMA ofr") + con.execute("CREATE TABLE ofr.parent(id INTEGER PRIMARY KEY, nm TEXT)") + con.execute( + "CREATE TABLE ofr.child(id INTEGER PRIMARY KEY, " + "parent_id INTEGER REFERENCES ofr.parent(id), amt INTEGER)" + ) + con.close() + + +class TestForeignKeysAreSchemaAware: + """The FK graph, the join generator and the FK-column collector all take + the DISCOVERY token, never the emitted qualifier. Nothing else in the + suite exercises a foreign key, so a token mix-up there would be invisible. + """ + + def test_joins_are_generated_for_a_non_default_schema(self, tmp_path): + db_path = str(tmp_path / "fk.duckdb") + _seed_fk(db_path) + models = _by_name( + ingest_datasource(datasource=_duckdb_ds(db_path), all_schemas=True) + ) + + assert models["child"].sql_table == "ofr.child" + assert models["parent"].sql_table == "ofr.parent" + assert [ + (j.target_model, [list(p) for p in j.join_pairs]) + for j in models["child"].joins + ] == [("parent", [["parent_id", "id"]])] + + def test_fk_columns_are_excluded_from_rollup(self, tmp_path): + """``_collect_fk_columns`` is the other consumer of the token; if it + silently returned nothing the FK column would be rolled up.""" + db_path = str(tmp_path / "fk.duckdb") + _seed_fk(db_path) + models = _by_name( + ingest_datasource(datasource=_duckdb_ds(db_path), all_schemas=True) + ) + child = models["child"] + assert {c.name for c in child.columns} >= {"id", "parent_id", "amt"} + + +class TestPrimaryKeysAreSchemaAware: + def test_primary_key_survives_a_qualified_schema_token(self, tmp_path): + """DuckDB's Inspector reports an empty ``constrained_columns`` even + for a declared PRIMARY KEY, so the ``INFORMATION_SCHEMA`` fallback is + the path that actually runs. It filters on ``table_schema``, which + holds the BARE name — a qualified token matches nothing and drops + every primary key silently. Fan-out safety leans on + ``Column.primary_key``, so losing it is not cosmetic. + """ + db_path = str(tmp_path / "fk.duckdb") + _seed_fk(db_path) + models = _by_name( + ingest_datasource(datasource=_duckdb_ds(db_path), all_schemas=True) + ) + + parent = models["parent"] + assert parent.sql_table == "ofr.parent" + assert [c.name for c in parent.columns if c.primary_key] == ["id"] + + def test_primary_key_survives_in_the_default_schema(self, tmp_path): + """The same fallback runs for the default schema, whose token is + qualified too (``fda.main``) even with nothing attached.""" + ds = _repro_ds(tmp_path) + db_path = ds.database + con = duckdb.connect(db_path) + con.execute("CREATE TABLE keyed(id INTEGER PRIMARY KEY, v INTEGER)") + con.close() + + models = _by_name(ingest_datasource(datasource=ds)) + assert [c.name for c in models["keyed"].columns if c.primary_key] == ["id"] + + +class TestPerObjectIsolation: + async def test_a_raising_column_lookup_becomes_a_skip( + self, tmp_path, monkeypatch + ): + """An ambiguous column lookup raises rather than guessing. That raise + must be isolated per object — one unresolvable table cannot abort the + scan — and must surface as a skip, not an error.""" + ds = _repro_ds(tmp_path) + real = ingestion_module._safe_get_columns + + def _raising(inspector, sa_engine, table_name, schema): + if table_name == "reports": + raise ValueError( + "ambiguous schema for 'reports': aaa.main, att_main.main" + ) + return real(inspector, sa_engine, table_name, schema) + + monkeypatch.setattr(ingestion_module, "_safe_get_columns", _raising) + + report = ingest_datasource_report(datasource=ds, all_schemas=True) + + assert "in_default" in _by_name(report.models) + assert "reports" not in _by_name(report.models) + assert any("ambiguous" in s.reason for s in report.skipped), ( + report.skipped + ) + + +class TestAdditiveMergeQualifierRules: + """Pinned directly on ``_additive_merge_existing``. Driving these through + ingestion would let the cross-schema guard skip the model before the merge + ran, so the merge rule itself would go untested.""" + + @staticmethod + def _model(sql_table: str, columns: list[str]) -> SlayerModel: + return SlayerModel( + name="reports", + data_source="ds", + sql_table=sql_table, + columns=[Column(name=c, type=DataType.INT) for c in columns], + ) + + def test_qualified_persisted_table_is_never_rewritten(self): + result = _additive_merge_existing( + persisted=self._model("prod.reports", ["id"]), + fresh=self._model("openfda_rest.reports", ["id", "n"]), + ) + assert result.merged.sql_table == "prod.reports" + assert result.sql_table_change is None + + def test_unqualified_persisted_table_is_healed(self): + result = _additive_merge_existing( + persisted=self._model("reports", ["id"]), + fresh=self._model("openfda_rest.reports", ["id"]), + ) + assert result.merged.sql_table == "openfda_rest.reports" + assert result.sql_table_change == "reports → openfda_rest.reports" + + def test_heal_alone_is_enough_to_trigger_a_save(self): + """A qualifier repair usually changes no columns and no joins, so if it + does not participate in the short-circuit the merged model is computed + and then discarded.""" + result = _additive_merge_existing( + persisted=self._model("reports", ["id"]), + fresh=self._model("openfda_rest.reports", ["id"]), + ) + assert result.new_columns == [] + assert result.new_joins == [] + assert result.merged is not None + assert result.sql_table_change is not None + + def test_a_different_object_name_is_not_a_qualifier_repair(self): + """Healing keys on the bare names matching. ``reports`` and + ``s.other`` are unrelated tables that happen to share a model name.""" + result = _additive_merge_existing( + persisted=self._model("reports", ["id"]), + fresh=self._model("openfda_rest.other", ["id"]), + ) + assert result.merged.sql_table == "reports" + assert result.sql_table_change is None + + +class TestAdditionRendering: + def test_updated_line_names_the_qualifier_repair(self): + """The repair is the whole point of the re-ingest, so it cannot be + silent — and a repair adds no columns, so without this the line prints + nothing at all.""" + buf = io.StringIO() + _print_ingest_addition( + ModelAddition( + model_name="reports", + data_source="ds", + created=False, + sql_table_change="reports → openfda_rest.reports", + ), + file=buf, + ) + out = buf.getvalue() + assert "reports → openfda_rest.reports" in out + assert "Updated" in out + + +class TestSystemSchemaFilter: + @pytest.mark.parametrize( + "token", + [ + "information_schema", + "INFORMATION_SCHEMA", + "pg_catalog", + "Pg_Catalog", + "pg_toast", + "performance_schema", + "mysql", + "sys", + "sys_temp", + "pg_temp_3", + "pg_toast_temp_1", + "system.main", + "system.information_schema", + "temp.main", + "TEMP.main", + ], + ) + def test_system_schemas_are_filtered(self, token): + assert _is_system_schema(token) is True + + @pytest.mark.parametrize( + "token", + [ + "main", + "public", + "openfda_rest", + "fda.main", + "fda.openfda_rest", + "systems", + "temporary", + "my_sys", + ], + ) + def test_user_schemas_are_kept(self, token): + assert _is_system_schema(token) is False + + +class TestCollisionTieBreakOrder: + def test_lower_schema_wins_and_order_does_not_matter(self): + """Rule 3. Both objects are sanitization-free and neither schema is the + default, so only the schema name can decide — and it must decide the + same way whichever order the inspector listed them in.""" + forward = [ + IngestableObject(name="reports", kind="table", schema="s2"), + IngestableObject(name="reports", kind="table", schema="s3"), + ] + reverse = list(reversed(forward)) + + for objects in (forward, reverse): + assigned, skipped = _assign_model_names(objects) + assert assigned[("s2", "reports")] == "reports" + assert ("s3", "reports") not in assigned + assert [s.table_name for s in skipped] == ["s3.reports"] + + def test_default_schema_beats_a_lower_sorted_schema(self): + """Rule 2 outranks rule 3: ``aaa`` sorts first but ``main`` is the + default, so a plain sort would pick the wrong winner.""" + objects = [ + IngestableObject(name="reports", kind="table", schema="aaa"), + IngestableObject(name="reports", kind="table", schema="main"), + ] + resolved = { + "aaa": ResolvedSchema(name="aaa", explicit=False, is_default=False), + "main": ResolvedSchema(name="main", explicit=False, is_default=True), + } + assigned, _ = _assign_model_names(objects, resolved_by_schema=resolved) + assert assigned[("main", "reports")] == "reports" + assert ("aaa", "reports") not in assigned + + +class TestMcpCreateDatasourceParity: + async def _call(self, storage, **kwargs) -> str: + server = create_mcp_server(storage=storage) + content, _ = await server.call_tool( + name="create_datasource", arguments=kwargs + ) + return content[0].text + + async def test_schema_name_is_persisted_and_used(self, tmp_path): + db_path = str(tmp_path / "fda.duckdb") + _seed_repro(db_path) + storage = _storage(tmp_path) + + await self._call( + storage, + name="ds", + type="duckdb", + database=db_path, + schema_name="openfda_rest", + ) + assert (await storage.get_datasource("ds")).schema_name == "openfda_rest" + assert ( + await storage.get_model("reports", data_source="ds") + ).sql_table == "openfda_rest.reports" + + async def test_schemas_csv_does_not_persist_schema_name(self, tmp_path): + db_path = str(tmp_path / "fda.duckdb") + _seed_repro(db_path) + storage = _storage(tmp_path) + + await self._call( + storage, + name="ds", + type="duckdb", + database=db_path, + schemas="main,openfda_rest", + ) + assert (await storage.get_datasource("ds")).schema_name is None + assert ( + await storage.get_model("reports", data_source="ds") + ).sql_table == "openfda_rest.reports" + assert ( + await storage.get_model("in_default", data_source="ds") + ).sql_table == "in_default" + + async def test_all_schemas_does_not_persist_schema_name(self, tmp_path): + db_path = str(tmp_path / "fda.duckdb") + _seed_repro(db_path) + storage = _storage(tmp_path) + + await self._call( + storage, + name="ds", + type="duckdb", + database=db_path, + all_schemas=True, + ) + assert (await storage.get_datasource("ds")).schema_name is None + assert ( + await storage.get_model("reports", data_source="ds") + ).sql_table == "openfda_rest.reports" + + @pytest.mark.parametrize( + "kwargs", + [ + {"schema_name": "a", "schemas": "b"}, + {"all_schemas": True, "schema_name": "a"}, + {"all_schemas": True, "schemas": "b"}, + ], + ) + async def test_conflicting_scope_arguments_are_reported( + self, tmp_path, kwargs + ): + db_path = str(tmp_path / "fda.duckdb") + _seed_repro(db_path) + storage = _storage(tmp_path) + + out = await self._call( + storage, name="ds", type="duckdb", database=db_path, **kwargs + ) + assert "schema" in out.lower() + assert any(w in out.lower() for w in ("cannot", "conflict", "both")) + + +class TestCliArgumentParsing: + """The parsers are built inline in ``main()``, so drive them through + ``main()`` with a stubbed handler that captures the parsed args.""" + + @staticmethod + def _capture(monkeypatch, argv: list[str], handler: str) -> SimpleNamespace: + captured: dict[str, SimpleNamespace] = {} + + def _stub(args, *_rest, **_kwargs): + captured["args"] = args + + monkeypatch.setattr(f"slayer.cli.{handler}", _stub) + monkeypatch.setattr(sys, "argv", ["slayer", *argv]) + + main() + return captured["args"] + + def test_ingest_schema_accepts_a_csv_list(self, monkeypatch): + args = self._capture( + monkeypatch, + ["ingest", "--datasource", "ds", "--schema", "main,openfda_rest"], + "_run_ingest", + ) + assert args.schema == "main,openfda_rest" + assert args.all_schemas is False + + def test_ingest_accepts_all_schemas(self, monkeypatch): + args = self._capture( + monkeypatch, + ["ingest", "--datasource", "ds", "--all-schemas"], + "_run_ingest", + ) + assert args.all_schemas is True + assert args.schema is None + + def test_ingest_defaults_all_schemas_off(self, monkeypatch): + args = self._capture( + monkeypatch, ["ingest", "--datasource", "ds"], "_run_ingest" + ) + assert args.all_schemas is False + + def test_ingest_rejects_schema_with_all_schemas(self, monkeypatch): + monkeypatch.setattr( + sys, + "argv", + [ + "slayer", "ingest", "--datasource", "ds", + "--schema", "main", "--all-schemas", + ], + ) + with pytest.raises(SystemExit) as excinfo: + main() + assert excinfo.value.code == 2 + + def test_datasources_create_accepts_all_schemas(self, monkeypatch): + args = self._capture( + monkeypatch, + ["datasources", "create", "duckdb:///x.duckdb", "--all-schemas"], + "_run_datasources_create", + ) + assert args.all_schemas is True + + def test_datasources_create_rejects_schema_with_all_schemas( + self, monkeypatch + ): + monkeypatch.setattr( + sys, + "argv", + [ + "slayer", "datasources", "create", "duckdb:///x.duckdb", + "--schema", "main", "--all-schemas", + ], + ) + with pytest.raises(SystemExit) as excinfo: + main() + assert excinfo.value.code == 2 + + +class TestRestLegacyCompatibility: + def _client(self, tmp_path, ds: DatasourceConfig): + storage = _storage(tmp_path) + client = TestClient(create_app(storage=storage)) + resp = client.post( + "/datasources", + json={"name": ds.name, "type": ds.type, "database": ds.database}, + ) + assert resp.status_code < 300, resp.text + return client, storage + + async def test_schema_name_is_still_honoured(self, tmp_path): + """The existing field keeps working — it folds to ``schemas=[value]`` + rather than being replaced.""" + ds = _repro_ds(tmp_path) + client, storage = self._client(tmp_path, ds) + resp = client.post( + "/ingest", json={"datasource": "ds", "schema_name": "openfda_rest"} + ) + assert resp.status_code == 200, resp.text + assert ( + await storage.get_model("reports", data_source="ds") + ).sql_table == "openfda_rest.reports" + + async def test_omitting_every_scope_field_uses_the_default_schema( + self, tmp_path + ): + ds = _repro_ds(tmp_path) + client, storage = self._client(tmp_path, ds) + resp = client.post("/ingest", json={"datasource": "ds"}) + assert resp.status_code == 200, resp.text + assert await storage.get_model("reports", data_source="ds") is None + assert ( + await storage.get_model("in_default", data_source="ds") + ).sql_table == "in_default" + + +class TestFallbackSqlShape: + """The catalog predicate must be added only when the token carries a + catalog, so every non-DuckDB dialect keeps emitting exactly today's SQL.""" + + @staticmethod + def _capture_sql(schema): + conn = MagicMock() + conn.execute.return_value.fetchall.return_value = [] + engine = MagicMock() + engine.connect.return_value.__enter__.return_value = conn + + _get_columns_fallback(engine, "orders", schema) + clause, params = conn.execute.call_args[0] + return str(clause), params + + def test_bare_schema_emits_no_catalog_predicate(self): + sql, params = self._capture_sql("public") + assert "table_catalog" not in sql + assert "catalog" not in params + assert params == {"table_name": "orders", "schema": "public"} + + def test_qualified_schema_emits_the_catalog_predicate(self): + sql, params = self._capture_sql("att_main.openfda_rest") + assert "table_catalog" in sql + assert params["catalog"] == "att_main" + assert params["schema"] == "openfda_rest" + + +# Re-exported for callers that patch discovery in place (the existing +# name-sanitization tests do this); asserting it here keeps that seam visible. +def test_module_exports_discovery_helpers(): + for attr in ( + "list_ingestable_objects", + "list_ingestable_objects_multi", + "resolve_ingest_schemas", + "qualify_sql_table", + "split_sql_table", + ): + assert hasattr(ingestion_module, attr), attr From 558c06dce838e3441e66e122238593a0566409b1 Mon Sep 17 00:00:00 2001 From: Egor Kraev Date: Fri, 7 Aug 2026 12:05:26 +0200 Subject: [PATCH 2/7] fix: resolve outside schema names before discovery; keep contested aliases MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Six findings from the Codex review of the PR diff, four of them defects I introduced, all reproduced against DuckDB before fixing. **Bare schema names still swept ATTACHed catalogs.** The token discipline was applied to the schemas we ENUMERATE but not to the ones handed to us: an explicit `--schema main`, a persisted `schema_name`, and the bare qualifier `validate-models` reads back off `sql_table` all went to the Inspector verbatim. Measured, `--schema main` on a database with a second catalog attached ingested that catalog's `only_in_other` and wrote it as `main.only_in_other`, which does not exist in the current catalog -- the exact bug class this branch exists to remove. `resolve_schema_token` upgrades such a name to the enumerated catalog-qualified token, preferring the current catalog when several expose the same schema name, and `ResolvedSchema` now carries `requested_as` so the emitted `sql_table` stays what the user typed. Resolving for discovery must never change the SQL we persist. **Dropping every contested alias was itself a data-loss bug.** The live map dropped a short alias claimed by more than one object, meaning to avoid an arbitrary winner. But default-schema models are persisted UNQUALIFIED by design, so as soon as another schema gained a same-named table, a legacy `sql_table: orders` stopped resolving -- and an unresolvable model is a `WholeModelDelete` that `validate-models --force-clean` deletes. A contested alias now resolves to the DEFAULT schema's entry, which is what the database itself does: `FROM orders` and `FROM main.orders` both land in the current catalog. It is dropped only when the default cannot break the tie. **The cross-schema guard failed open.** `_default_schema_object_names` converted a failed listing into an empty list, which reads as "no such default-schema object" and waved the qualifier repair through -- repointing a model at another schema's table. Unknown is now `None` and distinct from empty, and refuses the merge. Skipping a legal repair costs a re-run. **The PK fallback joined across catalogs.** DuckDB names a PK constraint after its column, so a same-shaped table in an ATTACHed catalog gets the identical auto-generated name; joining `key_column_usage` on constraint name and schema alone matched both and returned `['id', 'id']`. The join now carries the catalog. One finding rejected: emitting a 3-part `sql_table` for an explicitly-named catalog-qualified schema is by design, and verified queryable on DuckDB. Also from SonarQube: `# noqa: CODE — prose` is malformed suppression syntax (python:S7632), so the reasons move to their own line; the alias indexing is extracted into `_index_live_entries`, which also settles the cognitive complexity finding on `_live_schema_for_datasource`; and one composite test assertion is split. Three existing tests updated -- they pinned the pre-fix behaviour: the discovery token is now qualified where it used to be bare, and the contested alias resolves rather than missing. Co-Authored-By: Claude Opus 5 (1M context) --- DECISIONS.md | 2 +- slayer/engine/ingestion.py | 126 ++++++++++---- slayer/engine/introspect_utils.py | 67 ++++++- slayer/engine/schema_drift.py | 74 ++++++-- tests/test_ingestion_schema_qualification.py | 173 +++++++++++++++++-- 5 files changed, 374 insertions(+), 68 deletions(-) diff --git a/DECISIONS.md b/DECISIONS.md index 67f149bc..144390b1 100644 --- a/DECISIONS.md +++ b/DECISIONS.md @@ -70,4 +70,4 @@ implementation detail. Include issue refs when known. - 2026-08-04 — Dialect-aware / complete escaping for Mode-A `{variable}` substitution (DEV-1727), hardening DEV-1625. `substitute_variables(..., escape="sql")` is now **dialect-aware** and **fail-closed**: it gained a required keyword-only `backslash_escapes` signal (`bool | None`, raises if `None` in sql mode) so a caller rendering raw SQL can never silently under-escape. On backslash-escaping dialects (MySQL/ClickHouse/Snowflake/Redshift/BigQuery/Databricks/Spark) it doubles the backslash before escaping the single quote; on standard dialects it keeps the `''` quote-doubling. The double quote is deliberately left untouched — inside a single-quoted literal `\"` is NOT a recognised escape on 6 of the 7 backslash dialects (only MySQL), so escaping it would corrupt the value. The regime is DERIVED from sqlglot's own tokenizer via `SqlDialect.backslash_escapes_strings` (= `"\\" in tokenizer.STRING_ESCAPES`, guarded + 14-dialect pinned) so our escaping can never drift from the parser that reads the substituted SQL. `escape="python"` (Mode-B) additionally encodes the full C0 control range (`\t`/`\n`/`\r` named, rest `\xNN`) so raw newlines/NUL no longer break `ast.parse`. Engine fail-closed: `_substitute_model_sql_surfaces` / `_render_probe_model` require a `dialect`, threaded from the resolved datasource — no bare bool to forget. Assumes MySQL's default `sql_mode` (backslash escapes on); `NO_BACKSLASH_ESCAPES` servers are a sqlglot-layer-wide limitation, documented not fixed. The SQLite backslash end-to-end gap stays a pinned strict-xfail (pre-existing, out of scope). Bound parameters rejected (don't fit substitute-into-raw-SQL). Nested/join/cross-model lineages remain DEV-1678. - 2026-08-04 — Declared list-valued `{variable}` coercion (DEV-1730 follow-up): a scalar supplied for a variable the model declares `list_valued` is wrapped into a one-element list before Mode-A substitution, so an importer-generated `col IN ({var})` renders `IN ('US')` rather than the unquoted `IN (US)`. The generic scalar rule (author writes the quotes, so `{var}` also works in numeric/fragment positions like `amount >= {floor}` and `{d}::TIMESTAMP`) is CORRECT and unchanged — it just presumes an author who can see the SQL position, which a machine-generated fixed template does not have; the caller cannot supply per-element quotes through parentheses the importer wrote. Silent-wrong-answer risk drove the fix over a raise: `region IN (US)` parses as a column reference, so it fails at the database with a confusing message, or resolves against a real column and returns wrong rows. Opt-in is a front-end-NEUTRAL flag: the Cube converter writes `list_valued: ref.kind == "string"` into each `meta.cube_variables` entry (arrow forms splice pre-quoted scalars and stay `False`), and the engine reads only that flag — never Cube's `kind` taxonomy — so a future list-shaped front-end opts in the same way. Coercion lives at the single Mode-A choke point `_substitute_model_sql_surfaces` (execution and the `_render_probe_model` type-probe both route through it, so it cannot be bypassed) via `coerce_declared_list_variables` / `list_valued_variable_names` in `slayer/core/query.py`. Scope is deliberately narrow: only `str`/`int`/`float`/`bool` are wrapped; `list`/`tuple` pass through (the **empty list still raises** — "no filter" belongs to an optional block or a sentinel default); `None`/`dict` are left for `_render_variable_value` to reject with its own naming error; hand-written models declare nothing and are untouched. Follow-on from the same review: `declares_variables(model)` (any non-empty `meta.cube_variables`) now also defeats the DEV-1625 zero-variable fast path, via the shared `_model_needs_substitution_pass` predicate used by both `_substitute_model_sql_surfaces` and `_render_probe_model`. This closes the fast-path hole for a GENERATED model whose pushdowns are all required (no `{? ?}` block to force the pass): such a model used to emit a bare `{var}` into the SQL on a zero-variable call instead of raising the documented missing-variable error. The hole stays open — deliberately — for hand-written models, which declare nothing and keep the raw-brace-literal protection (`'{1,2,3}'`). The `list_valued` flag is matched with `is True`, not truthiness, since `meta` is user-extensible and a stray `1` or the string `"false"` must not switch substitution semantics. The bag is also SELF-IDENTIFYING — an entry counts only with a string `member` (the shape every importer writes) — so a hand-written `meta` that reuses the `cube_variables` key is not mistaken for generated SQL and silently stripped of its brace-literal protection. - 2026-08-05 — Ingestion sees views, survives unmodellable names, and stops being silent (DEV-1741). **Views**: `list_ingestable_objects` replaces the bare `get_table_names()` at every introspection site, adding `get_view_names` + `get_materialized_view_names` behind a `NotImplementedError`/`Exception` guard (the base `Inspector` RAISES for matviews on unsupporting dialects rather than returning `[]`), de-duplicated first-classification-wins because some dialects return views from `get_table_names()`, in a deterministic tables→views→matviews order that the name-collision policy depends on. Ingested by default — dbt materializes staging models as views, so opt-in would have left the reported failure in place for a fresh install — with `--no-views` on both `slayer ingest` and `datasources create --ingest`. The drift side (`_live_schema_for_datasource`) and the MCP listing (`_fetch_tables`) take **no flag and are unconditional**: that map is only ever a lookup target (`validate_datasource` iterates the *persisted* models, `available_in_ds` derives from them), so views there cannot manufacture a model or a drift entry — what they fix is a pre-existing **data-loss** bug where a hand-authored model whose `sql_table` named a view resolved to `live_table=None` → `WholeModelDelete` → deleted by `validate-models --force-clean`. Gating that on `--no-views` would re-arm it for exactly the users who opted out. **Names**: model names can't contain `__` (six modules — generator/enrichment/column_expansion/column_dependency/schema_drift/osi — `split("__")` an alias back into a join path, so a model named `a__b` is read as the alias for `a→b` and yields a silently wrong query, not a crash), but `sql_table` can, so a dlt child table `reports__patient__drug` is modelled as `reports_patient_drug` with `sql_table` verbatim. Sanitizer is `re.sub(r"_{2,}", "_")`, NOT `replace("__","_")` — `str.replace` is non-overlapping, so `a___b`→`a__b` would still fail validation. Collisions reserve every unsanitized name first (a real `a_b` always beats a sanitized `a__b`, order-independently) and **skip** rather than suffix, since suffixes shift as the object set changes and would orphan models and churn drift. A per-object `try/except` backstops everything else (`.`/`:`/`/`/`\` names, bad column names, per-object introspection failures); the FK-collection loop and `_get_fk_relationships` are guarded too because they run BEFORE that isolation and would otherwise still kill the run. **Reporting**: skips travel in a new `skipped` list, deliberately NOT folded into `errors` — `slayer ingest` exits 1 on either (we declined a perfectly valid object; `--exclude` is the documented remedy) but `POST /ingest` keeps 422 for `errors` only, because a permanent 422 aimed at a machine that can't act on the hint buries a successful partial ingest behind an error status. An empty scan prints the available schemas and exits 1 (the reporter listed the missing exit code as part of the defect), gated on `objects` not `additions` so a healthy no-op re-ingest stays quiet; `datasources create --ingest` on an empty DB still exits 0, since creating the datasource is that command's job and it succeeded. `in_scope_table_names` switched from model names to `_bare_table_name(sql_table)` — it is compared against table names in `_scoped_models_for_validation`, so the old keying silently dropped from validation scope any model whose name differs from its table (every sanitized model, and already every dbt/OSI hidden model passing `model_name=`). **`source_kind`** (`table`/`view`/`materialized_view`/`None`=unknown) persists on `SlayerModel`, v7→v8 with a no-op migration (mandatory: `migrate()` raises `RuntimeError` on an unregistered step); `None` for pre-v8, hand-authored and sql/query-backed models is the honest value, not a guess. It is a deliberate **exception to the additive-merge contract** — refreshed, not preserved — because it describes the live object rather than user intent, and the transition it exists to capture (dbt `+materialized: table`) usually changes no columns at all; the refresh therefore has to be made in three places (the early return, the `model_copy(update=...)`, and the save gate in `_process_one_table`), since editing only the update dict computes a corrected model and throws it away. A `None` from a non-classifying path never erases a known value. Docs-only fix for the advertised `motley-slayer[duckdb]` extra, which does not exist: `duckdb`/`duckdb-engine` are unconditional core deps (the Postgres facade imports duckdb at top level on every CLI invocation), and adding them under `[tool.poetry.extras]` would *gate* rather than alias them, breaking bare `pip install motley-slayer` → `datasources create demo`. -- 2026-08-07 — Ingest resolves a schema scope, and non-default schemas are written into `sql_table` (DEV-1758). **The bug**: `duckdb_engine`'s `get_table_names(schema=None)` returns objects from *every* schema as bare names — Postgres/MySQL/MSSQL/Snowflake/BigQuery/ClickHouse/SQLite all restrict it to the connection's default — so `_build_one_model`'s `f"{schema}.{name}" if schema else name` wrote an unqualified `sql_table` for a non-default-schema object and the generator emitted `FROM reports`, which fails table-not-found. The same schema-blindness made `_get_columns_fallback` union two same-named tables' columns. Not literally a #283 regression: that `sql_table` line is byte-identical before and after, and `--schema` always qualified correctly; #283 changed *visibility* by ingesting views by default, and dbt materialises staging models as views, so a dlt+dbt DuckDB file went from a handful of models to dozens, most unqueryable. **Scope**: one pass covers ONE schema — explicit `--schema a,b` / `--all-schemas` (plus `schemas` / `all_schemas` on the Python, REST and MCP surfaces), else `datasource.schema_name`, else the connection default. Multi-schema is opt-in because it changes what `sql_table` holds; when exactly one schema is scanned and others exist, the result carries a hint naming them and the exit code is unchanged (a hint is not a failure). `schema_name` is a **fallback**, never a conflict with an explicit flag; the genuine conflicts (`schema`+`schemas`, `all_schemas`+either) are rejected by one shared helper called from every entry point, so the CLI's mutually-exclusive group is not the only thing holding the line. **Two different strings**: the *discovery token* is carried exactly as `get_schema_names()` yields it — catalog-qualified on DuckDB (`fda.main`), bare elsewhere — and the qualified form is the SAFE one. Measured: with a second catalog `ATTACH`ed, a bare `main` makes `get_table_names` and `has_table` reach into the attached catalog and makes the column fallback return the cross-catalog union, while `att_main.main` is exact. So `is_default` compares the token IN FULL (no last-segment comparison, which is what made `att_main.main` and `other.main` both read as the default), and the INFORMATION_SCHEMA fallbacks filter on `table_catalog` as well as `table_schema` — `table_schema` alone holds the bare name, so a qualified token matched nothing and produced a silently **column-less** model, and retrying bare would resurrect the union. This applies to the PK fallback too, which on DuckDB is the path that actually runs (its Inspector reports no PK even for a declared PRIMARY KEY) — filtering it wrong drops every primary key, and fan-out safety leans on `Column.primary_key`. The *emitted qualifier* is a different string: the bare last segment, since the connection's current catalog is already correct. **What gets qualified** (D-9): only non-default schemas, so widening the scan never rewrites models already on disk and one datasource legitimately mixes both forms; a schema named explicitly as a single value is written verbatim, preserving today's `--schema public` → `public.orders`. A multi-schema list is deliberately NOT verbatim — listing the default alongside another schema would re-qualify every existing model. `--all-schemas` means the **current catalog only**; attached catalogs are dropped loudly with the exact `--schema .` invocation that ingests them, never silently. **Merging**: re-ingest heals a MISSING qualifier (participating in the short-circuit and the save gate, like `source_kind` — a repair usually changes no columns, so a merge that only edits the update dict computes the fix and discards it) but never rewrites an existing one. Two schemas' same-named tables are never fused into one model: a schema mismatch skips, and — the case a schema comparison alone cannot see, because D-9 persists default-schema models *unqualified* — a bare persisted `sql_table` that names a real default-schema object also skips, rather than being repointed at another schema's table by the heal. **Collisions** are resolved in ONE phase over final model names with a 4-key total order (unsanitized beats sanitized, then default schema, then schema name, then object name), not as successive passes, so the mixed case (`s1.a__b` sanitizing onto a real `s2.a_b`) is defined and the outcome never depends on inspector listing order. Losers skip, never suffix. **Validation** derives its schema set from the models being validated rather than gaining a flag, and the live map is keyed on the full `.` identity with shorter aliases inserted only when unique — an ambiguous alias is dropped so a lookup misses instead of resolving to another catalog's same-named table. This is a data-loss path, not a nuisance: an unresolvable model is a `WholeModelDelete` that `validate-models --force-clean` acts on. One dotted-name splitter (`split_sql_table`, everything before the FINAL dot) replaces three disagreeing parsers, so hand-written Snowflake `db.schema.table` and BigQuery `project.dataset.table` stop losing their catalog. No new model field and no v9 migration — the schema lives in `sql_table`, which is where the generator already reads it. +- 2026-08-07 — Ingest resolves a schema scope, and non-default schemas are written into `sql_table` (DEV-1758). **The bug**: `duckdb_engine`'s `get_table_names(schema=None)` returns objects from *every* schema as bare names — Postgres/MySQL/MSSQL/Snowflake/BigQuery/ClickHouse/SQLite all restrict it to the connection's default — so `_build_one_model`'s `f"{schema}.{name}" if schema else name` wrote an unqualified `sql_table` for a non-default-schema object and the generator emitted `FROM reports`, which fails table-not-found. The same schema-blindness made `_get_columns_fallback` union two same-named tables' columns. Not literally a #283 regression: that `sql_table` line is byte-identical before and after, and `--schema` always qualified correctly; #283 changed *visibility* by ingesting views by default, and dbt materialises staging models as views, so a dlt+dbt DuckDB file went from a handful of models to dozens, most unqueryable. **Scope**: one pass covers ONE schema — explicit `--schema a,b` / `--all-schemas` (plus `schemas` / `all_schemas` on the Python, REST and MCP surfaces), else `datasource.schema_name`, else the connection default. Multi-schema is opt-in because it changes what `sql_table` holds; when exactly one schema is scanned and others exist, the result carries a hint naming them and the exit code is unchanged (a hint is not a failure). `schema_name` is a **fallback**, never a conflict with an explicit flag; the genuine conflicts (`schema`+`schemas`, `all_schemas`+either) are rejected by one shared helper called from every entry point, so the CLI's mutually-exclusive group is not the only thing holding the line. **Two different strings**: the *discovery token* is carried exactly as `get_schema_names()` yields it — catalog-qualified on DuckDB (`fda.main`), bare elsewhere — and the qualified form is the SAFE one. Measured: with a second catalog `ATTACH`ed, a bare `main` makes `get_table_names` and `has_table` reach into the attached catalog and makes the column fallback return the cross-catalog union, while `att_main.main` is exact. So `is_default` compares the token IN FULL (no last-segment comparison, which is what made `att_main.main` and `other.main` both read as the default), and the INFORMATION_SCHEMA fallbacks filter on `table_catalog` as well as `table_schema` — `table_schema` alone holds the bare name, so a qualified token matched nothing and produced a silently **column-less** model, and retrying bare would resurrect the union. This applies to the PK fallback too, which on DuckDB is the path that actually runs (its Inspector reports no PK even for a declared PRIMARY KEY) — filtering it wrong drops every primary key, and fan-out safety leans on `Column.primary_key`. The *emitted qualifier* is a different string: the bare last segment, since the connection's current catalog is already correct. **What gets qualified** (D-9): only non-default schemas, so widening the scan never rewrites models already on disk and one datasource legitimately mixes both forms; a schema named explicitly as a single value is written verbatim, preserving today's `--schema public` → `public.orders`. A multi-schema list is deliberately NOT verbatim — listing the default alongside another schema would re-qualify every existing model. `--all-schemas` means the **current catalog only**; attached catalogs are dropped loudly with the exact `--schema .` invocation that ingests them, never silently. **Merging**: re-ingest heals a MISSING qualifier (participating in the short-circuit and the save gate, like `source_kind` — a repair usually changes no columns, so a merge that only edits the update dict computes the fix and discards it) but never rewrites an existing one. Two schemas' same-named tables are never fused into one model: a schema mismatch skips, and — the case a schema comparison alone cannot see, because D-9 persists default-schema models *unqualified* — a bare persisted `sql_table` that names a real default-schema object also skips, rather than being repointed at another schema's table by the heal. **Collisions** are resolved in ONE phase over final model names with a 4-key total order (unsanitized beats sanitized, then default schema, then schema name, then object name), not as successive passes, so the mixed case (`s1.a__b` sanitizing onto a real `s2.a_b`) is defined and the outcome never depends on inspector listing order. Losers skip, never suffix. **Validation** derives its schema set from the models being validated rather than gaining a flag, and the live map is keyed on the full `.` identity plus shorter aliases. A contested alias resolves to the DEFAULT schema's entry, mirroring what the database itself does (`FROM orders` and `FROM main.orders` both land in the current catalog); it is dropped only when the default cannot break the tie. Dropping every contested alias — the first cut, and what the plan review asked for — was itself a data-loss bug, because default-schema models are persisted UNQUALIFIED by design, so the moment another schema gained a same-named table the legacy model stopped resolving. Schema names arriving from outside — a `--schema` argument, a persisted `schema_name`, a bare qualifier read back off `sql_table` — are upgraded to the enumerated catalog-qualified token before they reach an Inspector, since a bare token is exactly what sweeps ATTACHed catalogs; what the user typed is kept separately and is what gets emitted, so resolving for discovery can never change the SQL persisted. This is a data-loss path, not a nuisance: an unresolvable model is a `WholeModelDelete` that `validate-models --force-clean` acts on. One dotted-name splitter (`split_sql_table`, everything before the FINAL dot) replaces three disagreeing parsers, so hand-written Snowflake `db.schema.table` and BigQuery `project.dataset.table` stop losing their catalog. No new model field and no v9 migration — the schema lives in `sql_table`, which is where the generator already reads it. diff --git a/slayer/engine/ingestion.py b/slayer/engine/ingestion.py index c041a14a..6776544c 100644 --- a/slayer/engine/ingestion.py +++ b/slayer/engine/ingestion.py @@ -27,13 +27,16 @@ SlayerModel, sanitize_model_name, ) -from slayer.engine.introspect_utils import ( # noqa: F401 (re-exported for back-compat) +from slayer.engine.introspect_utils import ( # noqa: F401 + # (re-exported for back-compat) _FLOAT_LIKE_INFO_SCHEMA_TYPES, _INFO_SCHEMA_TYPE_MAP, _get_columns_fallback, _parse_info_schema_is_float, _safe_get_columns, + enumerated_schema_names, qualified_default_schema, + resolve_schema_token, split_schema_token, split_sql_table, ) @@ -475,12 +478,17 @@ def _get_pk_constraint_fallback( if catalog is not None: clauses.append("tc.table_catalog = :catalog") params["catalog"] = catalog + # The join carries the catalog too: constraint names are only unique + # within a catalog, and DuckDB generates the same ``t_id_pkey`` for a + # same-shaped table in an ATTACHed one — which joined across catalogs + # and returned the PK column twice. sql = ( "SELECT kcu.column_name " "FROM information_schema.table_constraints tc " "JOIN information_schema.key_column_usage kcu " " ON tc.constraint_name = kcu.constraint_name " " AND tc.table_schema = kcu.table_schema " + " AND tc.table_catalog = kcu.table_catalog " "WHERE " + " AND ".join(clauses) ) else: @@ -847,8 +855,9 @@ class IngestionScanReport(BaseModel): # Object names living in the connection's default schema. The additive # pass needs it to tell "this persisted unqualified model IS the default # schema's table" from "a same-named table in another schema", which is - # the one case a schema comparison alone cannot decide. - default_schema_objects: list[str] = Field(default_factory=list) + # the one case a schema comparison alone cannot decide. ``None`` means the + # listing failed — distinct from empty, because the consumer fails closed. + default_schema_objects: list[str] | None = None # --------------------------------------------------------------------------- @@ -886,14 +895,22 @@ def _is_system_schema(token: str) -> bool: class ResolvedSchema(BaseModel): """One schema in ingest scope. - ``name`` is the *discovery* token, carried exactly as the dialect - enumerates it. ``explicit`` means the user named this single schema, so - its qualifier is written verbatim; a multi-schema request follows the - automatic rules instead, or listing the default schema alongside another - would re-qualify every model already on disk. + ``name`` is the *discovery* token, in the shape the dialect enumerates — + catalog-qualified on DuckDB. A schema the user named bare is upgraded to + that shape, because a bare token reaches into ``ATTACH``ed catalogs. + + ``requested_as`` is what the user actually typed, and is what gets written + into ``sql_table`` on the ``explicit`` path — resolving a token for + discovery must never change the SQL we emit. + + ``explicit`` means the user named this single schema, so its qualifier is + written verbatim; a multi-schema request follows the automatic rules + instead, or listing the default schema alongside another would re-qualify + every model already on disk. """ name: str | None = None + requested_as: str | None = None explicit: bool = False is_default: bool = False @@ -970,12 +987,32 @@ def _current_catalog_only( def _enumerate_schemas(inspector: sa.engine.Inspector) -> list[str]: """Every non-system schema the connection can see, tokens as enumerated.""" - try: - names = list(inspector.get_schema_names() or []) - except Exception as exc: # noqa: BLE001 — enumeration is best-effort - logger.debug("get_schema_names failed: %s", exc) - return [] - return sorted(n for n in names if isinstance(n, str) and not _is_system_schema(n)) + return sorted( + n for n in enumerated_schema_names(inspector) if not _is_system_schema(n) + ) + + +def _resolved_from_request( + *, + token: str, + inspector: sa.engine.Inspector, + enumerated: list[str], + default_token: str | None, + explicit: bool, +) -> ResolvedSchema: + """Build a :class:`ResolvedSchema` for a schema the user named. + + The discovery token is upgraded to the enumerated (catalog-qualified) + shape where that is unambiguous — a bare ``main`` otherwise sweeps + ``ATTACH``ed catalogs — while ``requested_as`` keeps the user's own string + so the emitted ``sql_table`` is unchanged by the upgrade. + """ + return ResolvedSchema( + name=resolve_schema_token(inspector, token, enumerated=enumerated), + requested_as=token, + explicit=explicit, + is_default=_matches_default(token, default_token), + ) def resolve_ingest_schemas( @@ -1011,19 +1048,23 @@ def resolve_ingest_schemas( # default schema verbatim there would rewrite every model on disk. single = len(requested) == 1 schemas = [ - ResolvedSchema( - name=token, + _resolved_from_request( + token=token, + inspector=inspector, + enumerated=enumerated, + default_token=default_token, explicit=single, - is_default=_matches_default(token, default_token), ) for token in requested ] elif datasource_schema: schemas = [ - ResolvedSchema( - name=datasource_schema, + _resolved_from_request( + token=datasource_schema, + inspector=inspector, + enumerated=enumerated, + default_token=default_token, explicit=True, - is_default=_matches_default(datasource_schema, default_token), ) ] else: @@ -1049,9 +1090,13 @@ def qualify_sql_table(*, obj: IngestableObject, resolved: ResolvedSchema) -> str Default-schema objects stay unqualified so that widening the scan never rewrites models that already exist. + + The explicit path emits ``requested_as`` — what the user typed — not the + resolved discovery token, so upgrading ``main`` to ``fda.main`` for safe + introspection cannot leak a catalog into the persisted SQL. """ if resolved.explicit: - return f"{resolved.name}.{obj.name}" + return f"{resolved.requested_as or resolved.name}.{obj.name}" if resolved.is_default or not resolved.name: return obj.name return f"{_bare_schema(resolved.name)}.{obj.name}" @@ -1468,12 +1513,18 @@ def _default_schema_object_names( scope: IngestSchemaScope, objects: list[IngestableObject], include_views: bool, -) -> list[str]: +) -> list[str] | None: """Object names living in the connection's default schema. Derived from the objects already discovered when the default schema is in scope; otherwise listed explicitly, which costs one extra catalog call on the only path that needs it. + + ``None`` means "could not be determined" and is deliberately distinct from + the empty list. The consumer is a guard against repointing a model at a + different physical table, so an unknown answer has to fail CLOSED — an + empty list would read as "no default-schema object of that name exists" + and wave the repoint through. """ default_token = qualified_default_schema(inspector) if any(s.name == default_token for s in scope.schemas): @@ -1487,9 +1538,13 @@ def _default_schema_object_names( include_views=include_views, ) ] - except Exception as exc: # noqa: BLE001 — the guard degrades, never aborts - logger.debug("default-schema listing failed: %s", exc) - return [] + # Unknown, not empty — see the docstring. + except Exception as exc: # noqa: BLE001 + logger.warning( + "could not list the default schema; cross-schema merges will be " + "refused rather than guessed: %s", exc, + ) + return None def ingest_datasource_report( @@ -1846,7 +1901,7 @@ def _cross_schema_conflict( model_name: str, persisted: SlayerModel, fresh: SlayerModel, - default_schema_objects: set[str], + default_schema_objects: set[str] | None, ) -> SkippedTable | None: """Refuse to merge two different schemas' tables into one model. @@ -1855,6 +1910,12 @@ def _cross_schema_conflict( cannot: a default-schema model is persisted *unqualified*, so a fresh qualified object with the same bare name looks like a repair when it is actually a different table. + + ``default_schema_objects=None`` means the default schema could not be + listed. That fails CLOSED: an unqualified persisted model is treated as a + possible default-schema table and the merge is refused. Skipping a legal + repair costs a re-run; guessing wrong repoints a model at another + schema's data. """ persisted_table = persisted.sql_table or "" fresh_table = fresh.sql_table or "" @@ -1864,8 +1925,8 @@ def _cross_schema_conflict( return None conflicting = persisted_schema is not None and persisted_schema != fresh_schema - shadows_default = ( - persisted_schema is None and persisted_table in default_schema_objects + shadows_default = persisted_schema is None and ( + default_schema_objects is None or persisted_table in default_schema_objects ) if not (conflicting or shadows_default): return None @@ -1915,7 +1976,7 @@ async def _process_one_table( model_name=table_name, persisted=persisted, fresh=fresh, - default_schema_objects=default_schema_objects or set(), + default_schema_objects=default_schema_objects, ) if conflict is not None: return ProcessTableOutcome(skipped=conflict) @@ -2052,7 +2113,12 @@ async def ingest_datasource_idempotent( ) fresh_models = scan.models skipped = list(scan.skipped) - default_schema_objects = set(scan.default_schema_objects) + # ``None`` (listing failed) is carried through, not flattened to empty — + # the merge guard fails closed on it. + default_schema_objects = ( + None if scan.default_schema_objects is None + else set(scan.default_schema_objects) + ) fresh_by_name = {m.name: m for m in fresh_models} # Keyed on the LIVE OBJECT name, not the model name. ``_scoped_models_for_validation`` # compares this against ``_bare_table_name(m.sql_table)``, so using model diff --git a/slayer/engine/introspect_utils.py b/slayer/engine/introspect_utils.py index 7d450b7a..90ba2062 100644 --- a/slayer/engine/introspect_utils.py +++ b/slayer/engine/introspect_utils.py @@ -108,7 +108,8 @@ def _current_catalog(inspector: sa.engine.Inspector) -> Optional[str]: catalog = conn.exec_driver_sql("SELECT current_database()").scalar() if isinstance(catalog, str) and catalog: return catalog - except Exception: # noqa: BLE001 — probe only; the URL stem is the fallback + # Probe only — the URL stem is the fallback. + except Exception: # noqa: BLE001 pass try: database = inspector.engine.url.database @@ -117,6 +118,59 @@ def _current_catalog(inspector: sa.engine.Inspector) -> Optional[str]: return Path(database).stem if database else None +def enumerated_schema_names(inspector: sa.engine.Inspector) -> list[str]: + """Every schema token the dialect enumerates, in its own shape.""" + try: + return [n for n in (inspector.get_schema_names() or []) if isinstance(n, str)] + # Enumeration is best-effort; callers fall back to the token as given. + except Exception: # noqa: BLE001 + return [] + + +def resolve_schema_token( + inspector: sa.engine.Inspector, + token: Optional[str], + *, + enumerated: Optional[list[str]] = None, +) -> Optional[str]: + """Resolve a user-supplied schema name to a safe *discovery* token. + + A user types ``--schema main``, and a persisted ``sql_table`` records the + bare ``analytics``. On DuckDB the bare form is the dangerous one — it makes + ``get_table_names`` and ``has_table`` reach into ``ATTACH``ed catalogs — so + a bare token is upgraded to the catalog-qualified token the dialect + enumerates, when exactly one matches. Ambiguous or unknown names are + returned unchanged: guessing between two catalogs would be worse than + letting the accessors report nothing. + + Dialects that enumerate bare names resolve to themselves, so this is a + no-op everywhere except DuckDB. + """ + if not token: + return token + names = enumerated if enumerated is not None else enumerated_schema_names(inspector) + if not names or token in names: + return token + if "." in token: + # Already carries a catalog and is not a known schema — honour it + # verbatim rather than second-guessing the user. + return token + matches = [n for n in names if n.rsplit(".", 1)[-1] == token] + if len(matches) == 1: + return matches[0] + if not matches: + return token + # Several catalogs expose a schema of this name. A bare name means the + # one in the database we are connected to — the same thing an unqualified + # ``FROM main.t`` resolves to — so prefer it over guessing or giving up. + catalog = _current_catalog(inspector) + if catalog: + in_catalog = [n for n in matches if n.split(".", 1)[0] == catalog] + if len(in_catalog) == 1: + return in_catalog[0] + return token + + def qualified_default_schema( inspector: sa.engine.Inspector, ) -> Optional[str]: @@ -138,7 +192,8 @@ def qualified_default_schema( token = _compute_default_schema_token(inspector) try: setattr(inspector, _DEFAULT_SCHEMA_ATTR, token) - except Exception: # noqa: BLE001 — caching is an optimisation, not a contract + # Caching is an optimisation, not a contract. + except Exception: # noqa: BLE001 pass return token @@ -148,13 +203,13 @@ def _compute_default_schema_token( ) -> Optional[str]: try: default = inspector.default_schema_name - except Exception: # noqa: BLE001 — a dialect may not implement it + # A dialect may not implement it. + except Exception: # noqa: BLE001 return None if not isinstance(default, str) or not default: return None - try: - names = list(inspector.get_schema_names() or []) - except Exception: # noqa: BLE001 — enumeration is best-effort + names = enumerated_schema_names(inspector) + if not names: return default if default in names: return default diff --git a/slayer/engine/schema_drift.py b/slayer/engine/schema_drift.py index f0339077..a36de5c3 100644 --- a/slayer/engine/schema_drift.py +++ b/slayer/engine/schema_drift.py @@ -15,7 +15,7 @@ import asyncio import logging -from collections import Counter +from collections import defaultdict from typing import ( Annotated, Any, @@ -1668,14 +1668,57 @@ def _alias_keys(schema_token: str | None, name: str) -> list[str]: A model written before the catalog was known says ``schema.table``; one written before schemas were recorded at all says just ``table``. Both must - keep resolving, so both aliases are offered — but only inserted when they - are unambiguous across everything scanned. + keep resolving, so both aliases are offered — but a contested alias is + only inserted for the entry the database itself would pick (see + :func:`_index_live_entries`). """ if not schema_token: return [name] return [f"{schema_token.rsplit('.', 1)[-1]}.{name}", name] +def _index_live_entries( + entries: list[tuple[str | None, str, LiveTable]], + *, + default_token: str | None, +) -> dict[str, LiveTable]: + """Key the live objects on their full identity, plus shorter aliases. + + Full ``"."`` keys are always present. A shorter + alias is inserted when exactly one object claims it, and — when several + do — for the one in the connection's DEFAULT schema, because that is + precisely how the database resolves the short form: ``FROM orders`` and + ``FROM main.orders`` both land in the current catalog's default schema. + + Dropping a contested alias outright looked safer but is not: an + unqualified ``sql_table`` is how every default-schema model is persisted, + so the moment another schema happened to hold a same-named table the + legacy model stopped resolving — and an unresolvable model is a + ``WholeModelDelete`` that ``validate-models --force-clean`` deletes. The + alias is only dropped when the default schema cannot break the tie, where + a miss really is better than an arbitrary winner. + """ + out: dict[str, LiveTable] = {} + for schema_token, name, live in entries: + out[f"{schema_token}.{name}" if schema_token else name] = live + + claimants: dict[str, list[tuple[str | None, str, LiveTable]]] = defaultdict(list) + for entry in entries: + for alias in _alias_keys(entry[0], entry[1]): + claimants[alias].append(entry) + + for alias, claiming in claimants.items(): + if alias in out: + continue + if len(claiming) == 1: + out[alias] = claiming[0][2] + continue + from_default = [e for e in claiming if e[0] == default_token] + if len(from_default) == 1: + out[alias] = from_default[0][2] + return out + + def _live_schema_for_datasource( *, datasource: DatasourceConfig, @@ -1707,12 +1750,20 @@ def _live_schema_for_datasource( data-loss bug for anyone who opted out of ingesting views. """ from slayer.engine.ingestion import _dispose_quietly, list_ingestable_objects + from slayer.engine.introspect_utils import ( + qualified_default_schema, + resolve_schema_token, + ) from slayer.sql import engine_factory sa_engine = engine_factory.get_engine(datasource.resolve_env_vars()) try: inspector = sa.inspect(sa_engine) entries: list[tuple[str | None, str, LiveTable]] = [] - for schema in schemas if schemas is not None else [None]: + for requested in schemas if schemas is not None else [None]: + # The schema set is derived from persisted ``sql_table`` values, + # which carry the BARE schema — and a bare token reaches into + # ATTACHed catalogs. Upgrade it to the enumerated shape first. + schema = resolve_schema_token(inspector, requested) for obj in list_ingestable_objects( inspector=inspector, schema=schema, include_views=True ): @@ -1735,20 +1786,9 @@ def _live_schema_for_datasource( datasource.name, exc, ) - - out: dict[str, LiveTable] = {} - for schema_token, name, live in entries: - out[f"{schema_token}.{name}" if schema_token else name] = live - alias_counts: Counter[str] = Counter( - alias - for schema_token, name, _ in entries - for alias in _alias_keys(schema_token, name) + return _index_live_entries( + entries, default_token=qualified_default_schema(inspector), ) - for schema_token, name, live in entries: - for alias in _alias_keys(schema_token, name): - if alias_counts[alias] == 1 and alias not in out: - out[alias] = live - return out finally: # Same rationale as ``ingest_datasource``: this is a one-shot # admin path. Disposing releases the underlying connection so diff --git a/tests/test_ingestion_schema_qualification.py b/tests/test_ingestion_schema_qualification.py index f84a5a90..0f5cf0a6 100644 --- a/tests/test_ingestion_schema_qualification.py +++ b/tests/test_ingestion_schema_qualification.py @@ -56,6 +56,8 @@ _additive_merge_existing, _assign_model_names, _bare_table_name, + _cross_schema_conflict, + _get_pk_constraint_fallback, _is_system_schema, _print_ingest_addition, _schema_of, @@ -1080,7 +1082,54 @@ def test_column_fallback_raises_when_it_cannot_disambiguate(self, attached): finally: eng.dispose() message = str(excinfo.value) - assert "aaa.main" in message and "att_main.main" in message + assert "aaa.main" in message + assert "att_main.main" in message + + def test_a_bare_requested_schema_is_upgraded_before_discovery( + self, attached + ): + """A user types ``--schema main``, and a bare token reaches into the + ATTACHed catalog exactly as ``schema=None`` used to. The token is + upgraded to the enumerated ``att_main.main`` for discovery — but the + emitted qualifier stays the ``main`` the user asked for, because + resolving for introspection must not change the SQL we persist.""" + report = ingest_datasource_report(datasource=attached.ds, schemas=["main"]) + + names = _by_name(report.models) + assert "only_in_other" not in names, sorted(names) + assert names["in_default"].sql_table == "main.in_default" + + def test_a_bare_schema_is_upgraded_for_validation_too(self, attached): + """``validate-models`` derives its schema set from persisted + ``sql_table`` values, which carry the BARE schema — so it hits the + same hazard from the other direction.""" + live = _live_schema_for_datasource( + datasource=attached.ds, schemas=["openfda_rest"] + ) + assert _resolve_live_table( + sql_table="openfda_rest.reports", live_tables=live + ) is not None + assert not any(k.startswith("aaa.") for k in live), sorted(live) + + def test_primary_key_is_not_duplicated_across_catalogs(self, tmp_path): + """DuckDB names a PK constraint after its column, so a same-shaped + table in an ATTACHed catalog gets the SAME auto-generated name. The + INFORMATION_SCHEMA join has to carry the catalog or it matches both + and returns the column twice.""" + main_path = str(tmp_path / "att_main.duckdb") + other_path = str(tmp_path / "att_other.duckdb") + for path, extra in ((main_path, "a"), (other_path, "b")): + con = duckdb.connect(path) + con.execute("CREATE SCHEMA ofr") + con.execute(f"CREATE TABLE ofr.t(id INTEGER PRIMARY KEY, {extra} INTEGER)") + con.close() + + eng = _attached_engine(main_path, other_path) + try: + pk = _get_pk_constraint_fallback(eng, "t", "att_main.ofr") + finally: + eng.dispose() + assert pk["constrained_columns"] == ["id"] def test_all_schemas_never_produces_a_columnless_model(self, attached): """Test 31. The failure mode this whole token discipline exists to @@ -1122,10 +1171,21 @@ def test_exact_name_beats_sanitized_regardless_of_schema_order( class TestLiveSchemaKeying: - def test_ambiguous_short_keys_are_dropped_not_overwritten(self, attached): + def test_contested_short_keys_resolve_the_way_the_database_would( + self, attached + ): """Test 36. Keying the live map on ``schema.table`` alone lets one - catalog's entry overwrite another's. Full keys are always present; - shorter aliases only when unambiguous.""" + catalog's entry overwrite another's, so full keys are always present + and shorter aliases are earned. + + A contested alias goes to the DEFAULT schema's entry, because that is + exactly what the database does: ``FROM shared`` and ``FROM + main.shared`` both land in the current catalog. Dropping the alias + instead looked safer and was not — every default-schema model is + persisted UNQUALIFIED, so the moment another catalog held a same-named + table the legacy model stopped resolving, and an unresolvable model is + a ``WholeModelDelete`` that ``--force-clean`` deletes. + """ live = _live_schema_for_datasource( datasource=attached.ds, schemas=["att_main.main", "aaa.main"], @@ -1133,12 +1193,51 @@ def test_ambiguous_short_keys_are_dropped_not_overwritten(self, attached): assert live["att_main.main.shared"].columns.keys() == {"m"} assert live["aaa.main.shared"].columns.keys() == {"o"} - # ``main.shared`` and ``shared`` are claimed by both, so neither alias - # may resolve to an arbitrary winner. - assert _resolve_live_table( - sql_table="main.shared", live_tables=live - ) is None - assert _resolve_live_table(sql_table="shared", live_tables=live) is None + + for short in ("main.shared", "shared"): + resolved = _resolve_live_table(sql_table=short, live_tables=live) + assert resolved is not None, short + assert resolved.columns.keys() == {"m"}, ( + f"{short} must resolve to the current catalog, not {short!r}'s " + f"namesake in the attached one" + ) + + async def test_a_legacy_unqualified_model_survives_a_same_named_table( + self, tmp_path + ): + """The data-loss path the alias rule exists to prevent, end to end: a + model persisted before schemas were recorded (``sql_table: orders``) + must keep resolving once another schema gains its own ``orders``. + Without the default-schema tie-break it becomes a ``WholeModelDelete`` + and ``validate-models --force-clean`` deletes it.""" + db_path = str(tmp_path / "legacy.duckdb") + con = duckdb.connect(db_path) + con.execute("CREATE TABLE orders(id INTEGER, amt INTEGER)") + con.execute("CREATE SCHEMA analytics") + con.execute("CREATE TABLE analytics.orders(x INTEGER, y INTEGER)") + con.close() + ds = _duckdb_ds(db_path) + + legacy = SlayerModel( + name="orders", data_source="ds", sql_table="orders", + columns=[ + Column(name="id", type=DataType.INT), + Column(name="amt", type=DataType.INT), + ], + ) + qualified = SlayerModel( + name="analytics_orders", data_source="ds", + sql_table="analytics.orders", + columns=[ + Column(name="x", type=DataType.INT), + Column(name="y", type=DataType.INT), + ], + ) + to_delete = await validate_datasource( + datasource=ds, models=[legacy, qualified] + ) + assert [e for e in to_delete if isinstance(e, WholeModelDelete)] == [] + assert to_delete == [], to_delete def test_unambiguous_aliases_are_still_inserted(self, attached): """The alias keys are what let an unqualified legacy model keep @@ -1255,9 +1354,15 @@ def test_requested_schemas_are_marked_explicit(self, tmp_path): finally: eng.dispose() - assert [(s.name, s.explicit) for s in scope.schemas] == [ - ("openfda_rest", True) - ] + # ``name`` is the DISCOVERY token, upgraded to the catalog-qualified + # shape the dialect enumerates — a bare token would reach into an + # ATTACHed catalog. ``requested_as`` keeps what the user typed, and is + # what gets emitted, so the upgrade cannot leak into ``sql_table``. + assert [ + (s.name.rsplit(".", 1)[-1], s.requested_as, s.explicit) + for s in scope.schemas + ] == [("openfda_rest", "openfda_rest", True)] + assert scope.schemas[0].name.endswith(".openfda_rest") def test_multi_schema_scope_reports_no_hint(self, tmp_path): """The hint is for "you may be missing something". With more than one @@ -1289,7 +1394,9 @@ def test_multi_returns_objects_tagged_with_their_schema(self, tmp_path): finally: eng.dispose() - assert {(o.name, o.schema) for o in objects} == { + # Objects carry the resolved discovery token, so the bare schema the + # user asked for shows up qualified here. + assert {(o.name, o.schema.rsplit(".", 1)[-1]) for o in objects} == { ("in_default", "main"), ("reports", "openfda_rest"), } @@ -1422,6 +1529,44 @@ def _raising(inspector, sa_engine, table_name, schema): ) +class TestCrossSchemaGuardFailsClosed: + """The guard answers "is this persisted unqualified model the default + schema's table?" from a live listing. When that listing fails the answer + is UNKNOWN, and unknown has to refuse the merge — an empty list would read + as "no such default-schema object" and wave a repoint through.""" + + @staticmethod + def _models() -> tuple[SlayerModel, SlayerModel]: + persisted = SlayerModel( + name="reports", data_source="ds", sql_table="reports", + columns=[Column(name="a", type=DataType.INT)], + ) + fresh = SlayerModel( + name="reports", data_source="ds", sql_table="s2.reports", + columns=[Column(name="b", type=DataType.INT)], + ) + return persisted, fresh + + def test_unknown_default_schema_refuses_the_merge(self): + persisted, fresh = self._models() + conflict = _cross_schema_conflict( + model_name="reports", persisted=persisted, fresh=fresh, + default_schema_objects=None, + ) + assert conflict is not None + assert "cross-schema" in conflict.reason + + def test_a_known_empty_default_schema_still_allows_the_repair(self): + """Fail-closed must not become fail-always: when the listing + succeeded and genuinely holds no such object, the qualifier repair is + the whole point of the re-ingest.""" + persisted, fresh = self._models() + assert _cross_schema_conflict( + model_name="reports", persisted=persisted, fresh=fresh, + default_schema_objects=set(), + ) is None + + class TestAdditiveMergeQualifierRules: """Pinned directly on ``_additive_merge_existing``. Driving these through ingestion would let the cross-schema guard skip the model before the merge From fb4762e1b48b7742db3ed9f202b3bb3babe3f5e8 Mon Sep 17 00:00:00 2001 From: Egor Kraev Date: Fri, 7 Aug 2026 13:16:37 +0200 Subject: [PATCH 3/7] fix: dedupe resolved schemas; gate token upgrade on qualifying dialects Two findings from review round 2, both reproduced first. **Two requests naming the same schema cancelled each other out.** `validate-models` derives its schema set from persisted `sql_table` values, so a datasource holding both `orders` (bare ingest) and `main.customers` (`--schema main`) asks for `None` AND `main` -- which now resolve to the same discovery token. Scanned twice, every object appeared as two rival claimants for its own alias, the default-schema tie-break found no unique winner, and the aliases were dropped from objects that have no rival at all. Measured: BOTH models became WholeModelDeletes, i.e. the fix for the previous round's data-loss bug had opened a wider one. Resolved tokens are now deduplicated before scanning, and `_index_live_entries` collapses duplicate `(schema_token, object)` pairs so the helper is correct whatever it is fed. **The catalog upgrade assumed every dot is a catalog separator.** Postgres allows `CREATE SCHEMA "foo.bar"` and lists schema names bare, so a request for a nonexistent `bar` would have silently resolved to `foo.bar` and ingested a schema the user never asked for. The upgrade is now gated on the dialect actually enumerating catalog-qualified tokens, decided from its own default schema token rather than from "does any name contain a dot". Co-Authored-By: Claude Opus 5 (1M context) --- slayer/engine/introspect_utils.py | 17 +++ slayer/engine/schema_drift.py | 37 ++++-- tests/test_ingestion_schema_qualification.py | 116 ++++++++++++++++++- 3 files changed, 158 insertions(+), 12 deletions(-) diff --git a/slayer/engine/introspect_utils.py b/slayer/engine/introspect_utils.py index 90ba2062..e95bf64d 100644 --- a/slayer/engine/introspect_utils.py +++ b/slayer/engine/introspect_utils.py @@ -127,6 +127,17 @@ def enumerated_schema_names(inspector: sa.engine.Inspector) -> list[str]: return [] +def _enumerates_qualified_tokens(inspector: sa.engine.Inspector) -> bool: + """Whether this dialect's schema tokens carry a catalog segment. + + Decided from the dialect's own default-schema token rather than from "does + any name contain a dot", so a dialect that merely permits a dot inside a + schema name is never mistaken for a catalog-qualifying one. + """ + default_token = qualified_default_schema(inspector) + return bool(default_token) and "." in default_token + + def resolve_schema_token( inspector: sa.engine.Inspector, token: Optional[str], @@ -155,6 +166,12 @@ def resolve_schema_token( # Already carries a catalog and is not a known schema — honour it # verbatim rather than second-guessing the user. return token + if not _enumerates_qualified_tokens(inspector): + # This dialect lists bare schema names, so a dot in one is part of the + # name (Postgres allows `CREATE SCHEMA "foo.bar"`), not a catalog + # separator. Reading it as one would silently ingest a schema the user + # did not ask for; an unknown name should simply find nothing. + return token matches = [n for n in names if n.rsplit(".", 1)[-1] == token] if len(matches) == 1: return matches[0] diff --git a/slayer/engine/schema_drift.py b/slayer/engine/schema_drift.py index a36de5c3..0ef6924a 100644 --- a/slayer/engine/schema_drift.py +++ b/slayer/engine/schema_drift.py @@ -1698,24 +1698,33 @@ def _index_live_entries( alias is only dropped when the default schema cannot break the tie, where a miss really is better than an arbitrary winner. """ - out: dict[str, LiveTable] = {} + # Deduplicate on identity first. Two requested schemas can resolve to the + # same discovery token (``None`` and ``main`` both become ``fda.main``), + # and a repeat would otherwise look like two rival claimants for every + # alias — silently dropping the aliases of objects that have no rival at + # all, which is the very deletion path this indexing exists to prevent. + unique: dict[tuple[str | None, str], LiveTable] = {} for schema_token, name, live in entries: + unique.setdefault((schema_token, name), live) + + out: dict[str, LiveTable] = {} + for (schema_token, name), live in unique.items(): out[f"{schema_token}.{name}" if schema_token else name] = live - claimants: dict[str, list[tuple[str | None, str, LiveTable]]] = defaultdict(list) - for entry in entries: - for alias in _alias_keys(entry[0], entry[1]): - claimants[alias].append(entry) + claimants: dict[str, list[tuple[str | None, str]]] = defaultdict(list) + for key in unique: + for alias in _alias_keys(key[0], key[1]): + claimants[alias].append(key) for alias, claiming in claimants.items(): if alias in out: continue if len(claiming) == 1: - out[alias] = claiming[0][2] + out[alias] = unique[claiming[0]] continue - from_default = [e for e in claiming if e[0] == default_token] + from_default = [k for k in claiming if k[0] == default_token] if len(from_default) == 1: - out[alias] = from_default[0][2] + out[alias] = unique[from_default[0]] return out @@ -1759,11 +1768,17 @@ def _live_schema_for_datasource( try: inspector = sa.inspect(sa_engine) entries: list[tuple[str | None, str, LiveTable]] = [] + # The schema set is derived from persisted ``sql_table`` values, which + # carry the BARE schema — and a bare token reaches into ATTACHed + # catalogs. Upgrade each to the enumerated shape, then dedupe: `None` + # and `main` both resolve to `fda.main`, and scanning it twice would + # double every entry. + scanned: list[str | None] = [] for requested in schemas if schemas is not None else [None]: - # The schema set is derived from persisted ``sql_table`` values, - # which carry the BARE schema — and a bare token reaches into - # ATTACHed catalogs. Upgrade it to the enumerated shape first. schema = resolve_schema_token(inspector, requested) + if schema not in scanned: + scanned.append(schema) + for schema in scanned: for obj in list_ingestable_objects( inspector=inspector, schema=schema, include_views=True ): diff --git a/tests/test_ingestion_schema_qualification.py b/tests/test_ingestion_schema_qualification.py index 0f5cf0a6..e32d3483 100644 --- a/tests/test_ingestion_schema_qualification.py +++ b/tests/test_ingestion_schema_qualification.py @@ -70,7 +70,11 @@ resolve_ingest_schemas, split_sql_table, ) -from slayer.engine.introspect_utils import _get_columns_fallback, _safe_get_columns +from slayer.engine.introspect_utils import ( + _get_columns_fallback, + _safe_get_columns, + resolve_schema_token, +) from slayer.engine.query_engine import SlayerQueryEngine from slayer.engine.schema_drift import ( LiveTable, @@ -1147,6 +1151,74 @@ def test_all_schemas_never_produces_a_columnless_model(self, attached): # --------------------------------------------------------------------------- +class TestSchemaTokenResolution: + def test_a_bare_token_is_upgraded_on_a_qualifying_dialect(self, tmp_path): + ds = _repro_ds(tmp_path) + eng, insp = _inspector_for(ds) + try: + assert resolve_schema_token(insp, "openfda_rest") == "fda.openfda_rest" + # Already qualified, and already what the dialect enumerates. + assert resolve_schema_token(insp, "fda.main") == "fda.main" + finally: + eng.dispose() + + def test_an_unknown_name_is_left_alone(self, tmp_path): + """An unknown schema should find nothing, not silently become a + different one.""" + ds = _repro_ds(tmp_path) + eng, insp = _inspector_for(ds) + try: + assert resolve_schema_token(insp, "nope") == "nope" + # A catalog-qualified name we do not recognise stays verbatim + # rather than being second-guessed. + assert resolve_schema_token(insp, "other.main") == "other.main" + finally: + eng.dispose() + + def test_a_dot_in_a_bare_dialects_schema_name_is_not_a_catalog(self): + """Postgres allows ``CREATE SCHEMA "foo.bar"`` and lists schema names + BARE, so a dot there belongs to the name. Reading it as a catalog + separator would silently resolve a request for the nonexistent ``bar`` + into ``foo.bar`` and ingest a schema the user never asked for. + + Mocked because no dialect SLayer tests against can hold both + properties at once — which is exactly why it needs pinning. + """ + insp = MagicMock(spec=sa.engine.Inspector) + insp.get_schema_names.return_value = ["public", "foo.bar"] + insp.default_schema_name = "public" + + assert resolve_schema_token(insp, "bar") == "bar" + assert resolve_schema_token(insp, "public") == "public" + + def test_sqlite_tokens_pass_through_unchanged(self, tmp_path): + db_path = str(tmp_path / "plain.db") + conn = sqlite3.connect(db_path) + conn.executescript("CREATE TABLE t (id INTEGER);") + conn.commit() + conn.close() + + eng = sa.create_engine(f"sqlite:///{db_path}", poolclass=StaticPool) + try: + insp = sa.inspect(eng) + assert resolve_schema_token(insp, "main") == "main" + assert resolve_schema_token(insp, "anything") == "anything" + finally: + eng.dispose() + + def test_the_current_catalog_wins_when_several_catalogs_match( + self, attached + ): + """``aaa`` and ``att_main`` both expose ``main``. A bare ``main`` + means the database we are connected to — the same thing the engine + does for an unqualified reference.""" + eng = attached.engine() + try: + assert resolve_schema_token(sa.inspect(eng), "main") == "att_main.main" + finally: + eng.dispose() + + class TestCollisionWithSanitization: @pytest.mark.parametrize("reverse", [False, True]) def test_exact_name_beats_sanitized_regardless_of_schema_order( @@ -1202,6 +1274,48 @@ def test_contested_short_keys_resolve_the_way_the_database_would( f"namesake in the attached one" ) + async def test_two_requests_naming_the_same_schema_do_not_cancel_out( + self, tmp_path + ): + """``validate-models`` derives its schema set from persisted + ``sql_table`` values, so a datasource holding both ``orders`` (bare + ingest) and ``main.customers`` (``--schema main``) asks for ``None`` + AND ``main`` — which resolve to the same discovery token. + + Scanned twice, every object appeared as two rival claimants for its + own alias, so the tie-break saw no unique winner and dropped the + aliases of objects that have no rival at all. Both models then became + ``WholeModelDelete``s. + """ + db_path = str(tmp_path / "dupe.duckdb") + con = duckdb.connect(db_path) + con.execute("CREATE TABLE orders(id INTEGER, amt INTEGER)") + con.execute("CREATE TABLE customers(cid INTEGER)") + con.close() + ds = _duckdb_ds(db_path) + + live = _live_schema_for_datasource(datasource=ds, schemas=[None, "main"]) + for short in ("orders", "main.customers"): + assert _resolve_live_table( + sql_table=short, live_tables=live + ) is not None, f"{short} lost its alias to a duplicate scan" + + models = [ + SlayerModel( + name="orders", data_source="ds", sql_table="orders", + columns=[ + Column(name="id", type=DataType.INT), + Column(name="amt", type=DataType.INT), + ], + ), + SlayerModel( + name="customers", data_source="ds", sql_table="main.customers", + columns=[Column(name="cid", type=DataType.INT)], + ), + ] + to_delete = await validate_datasource(datasource=ds, models=models) + assert [e for e in to_delete if isinstance(e, WholeModelDelete)] == [] + async def test_a_legacy_unqualified_model_survives_a_same_named_table( self, tmp_path ): From 6a3d49ef4ad10de619156d94fe706e5c7b8b5a49 Mon Sep 17 00:00:00 2001 From: Egor Kraev Date: Fri, 7 Aug 2026 13:21:59 +0200 Subject: [PATCH 4/7] fix: decide the qualifying-dialect gate from the enumeration, not the fallback Two findings from review round 3. **The gate could misclassify DuckDB as bare.** It asked `qualified_default_schema()`, which falls back to the BARE default when the current catalog cannot be determined -- and with attached catalogs supplying several `*.main` tokens, that fallback fires. The gate then reported "this dialect lists bare names", refused the upgrade, and re-armed the cross-catalog sweep the upgrade exists to prevent. It now asks where the dialect's own default schema turns up in its own enumeration: listed bare means bare tokens; absent but present as some `.` means the dialect qualifies. That is independent of catalog detection, and still keeps Postgres' `CREATE SCHEMA "foo.bar"` from being read as a catalog. **`None` was not normalised before the scan dedupe.** `None` and an explicit `main` resolved to different values (`None` stays `None`) even though `list_ingestable_objects` resolves both to the same token internally, so the schema was introspected twice and correctness rested entirely on the entry-level dedupe behind it. `None` now normalises to the default token up front. Co-Authored-By: Claude Opus 5 (1M context) --- slayer/engine/introspect_utils.py | 31 +++++++++++++++----- slayer/engine/schema_drift.py | 10 ++++++- tests/test_ingestion_schema_qualification.py | 26 +++++++++++++++- 3 files changed, 58 insertions(+), 9 deletions(-) diff --git a/slayer/engine/introspect_utils.py b/slayer/engine/introspect_utils.py index e95bf64d..967dbb1d 100644 --- a/slayer/engine/introspect_utils.py +++ b/slayer/engine/introspect_utils.py @@ -128,14 +128,31 @@ def enumerated_schema_names(inspector: sa.engine.Inspector) -> list[str]: def _enumerates_qualified_tokens(inspector: sa.engine.Inspector) -> bool: - """Whether this dialect's schema tokens carry a catalog segment. - - Decided from the dialect's own default-schema token rather than from "does - any name contain a dot", so a dialect that merely permits a dot inside a - schema name is never mistaken for a catalog-qualifying one. + """Whether this dialect prefixes its schema tokens with a catalog. + + Decided by asking where the dialect's OWN default schema turns up in its + OWN enumeration: listed bare means bare tokens (SQLite, Postgres, MySQL, + BigQuery); absent but present as some ``.`` means the + dialect qualifies (DuckDB). + + Deliberately not "does any enumerated name contain a dot" — Postgres + allows ``CREATE SCHEMA "foo.bar"``, and reading that as a catalog would + let it capture a request for a nonexistent ``bar``. Deliberately not + ``qualified_default_schema()`` either: that helper falls back to the bare + default when the current catalog cannot be determined, which would + misreport DuckDB as bare and re-arm the cross-catalog sweep. """ - default_token = qualified_default_schema(inspector) - return bool(default_token) and "." in default_token + try: + default = inspector.default_schema_name + # A dialect may not implement it. + except Exception: # noqa: BLE001 + return False + if not isinstance(default, str) or not default: + return False + names = enumerated_schema_names(inspector) + if not names or default in names: + return False + return any(n.rsplit(".", 1)[-1] == default for n in names) def resolve_schema_token( diff --git a/slayer/engine/schema_drift.py b/slayer/engine/schema_drift.py index 0ef6924a..0aafa1ca 100644 --- a/slayer/engine/schema_drift.py +++ b/slayer/engine/schema_drift.py @@ -1774,8 +1774,16 @@ def _live_schema_for_datasource( # and `main` both resolve to `fda.main`, and scanning it twice would # double every entry. scanned: list[str | None] = [] + default_token = qualified_default_schema(inspector) for requested in schemas if schemas is not None else [None]: - schema = resolve_schema_token(inspector, requested) + # ``None`` means the default schema, which is what + # ``list_ingestable_objects`` resolves it to anyway — normalise it + # here so it dedupes against an explicit request for that same + # schema instead of introspecting everything twice. + schema = ( + default_token if requested is None + else resolve_schema_token(inspector, requested) + ) if schema not in scanned: scanned.append(schema) for schema in scanned: diff --git a/tests/test_ingestion_schema_qualification.py b/tests/test_ingestion_schema_qualification.py index e32d3483..c8f86a22 100644 --- a/tests/test_ingestion_schema_qualification.py +++ b/tests/test_ingestion_schema_qualification.py @@ -31,7 +31,7 @@ import sys from pathlib import Path from types import SimpleNamespace -from unittest.mock import MagicMock +from unittest.mock import MagicMock, patch import pytest import sqlalchemy as sa @@ -1206,6 +1206,21 @@ def test_sqlite_tokens_pass_through_unchanged(self, tmp_path): finally: eng.dispose() + def test_a_qualifying_dialect_is_recognised_without_a_current_catalog( + self, + ): + """The gate must not be decided by ``qualified_default_schema``, which + falls back to the BARE default when the current catalog cannot be + determined. On DuckDB with attached catalogs that fallback would + misreport the dialect as bare and re-arm the cross-catalog sweep — so + the gate asks where the dialect's own default turns up in its own + enumeration instead.""" + insp = MagicMock(spec=sa.engine.Inspector) + insp.default_schema_name = "main" + insp.get_schema_names.return_value = ["aaa.main", "att_main.main", "att_main.ofr"] + + assert resolve_schema_token(insp, "ofr") == "att_main.ofr" + def test_the_current_catalog_wins_when_several_catalogs_match( self, attached ): @@ -1300,6 +1315,15 @@ async def test_two_requests_naming_the_same_schema_do_not_cancel_out( sql_table=short, live_tables=live ) is not None, f"{short} lost its alias to a duplicate scan" + # `None` normalises to the same discovery token as `main`, so the + # schema is introspected once, not twice. + with patch( + "slayer.engine.ingestion.list_ingestable_objects", + side_effect=ingestion_module.list_ingestable_objects, + ) as listed: + _live_schema_for_datasource(datasource=ds, schemas=[None, "main"]) + assert listed.call_count == 1, listed.call_args_list + models = [ SlayerModel( name="orders", data_source="ds", sql_table="orders", From 757d2d4200e62c9b6fbccd53ef6aafb6dfd0edf8 Mon Sep 17 00:00:00 2001 From: Egor Kraev Date: Tue, 11 Aug 2026 21:11:59 +0200 Subject: [PATCH 5/7] docs: trim verbose docstrings and comments (concise-comments pass) --- slayer/api/server.py | 19 +- slayer/cli.py | 20 +- slayer/core/enums.py | 8 +- slayer/core/models.py | 18 +- slayer/engine/ingestion.py | 317 +++++++---------- slayer/engine/introspect_utils.py | 95 ++--- slayer/engine/schema_drift.py | 148 ++++---- slayer/mcp/server.py | 19 +- slayer/storage/type_refinement.py | 5 +- slayer/storage/v8_migration.py | 16 +- tests/test_docs_duckdb_install.py | 13 +- tests/test_ingest_cli_ux.py | 29 +- tests/test_ingestion.py | 6 +- tests/test_ingestion_name_sanitize.py | 82 ++--- tests/test_ingestion_schema_qualification.py | 343 ++++++------------- tests/test_ingestion_views.py | 56 +-- tests/test_migrations.py | 5 +- tests/test_model_source_kind.py | 50 +-- tests/test_v7_migration.py | 9 +- 19 files changed, 451 insertions(+), 807 deletions(-) diff --git a/slayer/api/server.py b/slayer/api/server.py index e5caa9a3..8fa35386 100644 --- a/slayer/api/server.py +++ b/slayer/api/server.py @@ -110,9 +110,9 @@ class IngestRequest(BaseModel): @model_validator(mode="after") def _one_way_to_say_it(self) -> "IngestRequest": - """Reject conflicting scope arguments rather than silently preferring - whichever the handler reads first. The rule lives here so it applies - to every caller of the endpoint, and mirrors the engine's.""" + """Reject conflicting scope arguments at the edge, so every caller of + the endpoint gets the engine's rule (not whichever the handler reads + first).""" from slayer.engine.ingestion import _resolve_scope_args _resolve_scope_args( @@ -696,14 +696,11 @@ async def ingest(request: IngestRequest) -> dict[str, Any]: # Mirror the CLI's exit-1 behaviour by surfacing 422 with the # full IdempotentIngestResult body (additions/to_delete/errors). # - # ``result.skipped`` deliberately does NOT trigger 422, - # even though it DOES make `slayer ingest` exit 1. The divergence - # is intentional: a CLI exit code is a nag aimed at a human or a - # build log, and it has an obvious remedy (`--exclude `). A - # REST client cannot act on that hint, and a datasource with one - # permanently unmodellable object would otherwise return 422 on - # every ingest forever, burying a successful partial ingest behind - # an error status. Skips travel in the 200 body instead. + # ``result.skipped`` does NOT trigger 422 (though it makes + # `slayer ingest` exit 1): a REST client can't act on the + # ``--exclude`` hint, and a permanently-unmodellable object would + # otherwise return 422 forever, burying successful partial ingests. + # Skips travel in the 200 body instead. raise HTTPException( status_code=422, detail=result.model_dump(mode="json") ) diff --git a/slayer/cli.py b/slayer/cli.py index ea1a89a5..3f3104a2 100644 --- a/slayer/cli.py +++ b/slayer/cli.py @@ -1431,10 +1431,8 @@ def _run_ingest(args): ) ) - # An ingest that found nothing used to print nothing and exit 0, - # which is what convinced the reporter their schema of dbt views was empty. - # Gate on ``objects`` rather than ``additions`` so a healthy no-op re-ingest - # (objects exist, all already in sync) stays quiet. + # An empty ingest used to print nothing and exit 0. Gate on ``objects``, + # not ``additions``, so a healthy no-op re-ingest (all in sync) stays quiet. if not ( result.additions or result.to_delete @@ -1462,8 +1460,8 @@ def _run_ingest(args): # which schemas were left out. Advisory only — the exit code is unchanged. if getattr(result, "schema_hint", None): print(f"\n{result.schema_hint}") - # A skip means we declined to ingest a perfectly valid object, so it fails - # the command — `--exclude ` is the documented way to make it green. + # A skip declined a valid object, so it fails the command — `--exclude + # ` is the documented way to make it green. if result.errors or result.skipped: sys.exit(1) @@ -2004,9 +2002,8 @@ def _run_datasources_create(args, storage): name = args.name or derived_name schemas = _parse_csv_arg(args.schema) all_schemas = getattr(args, "all_schemas", False) - # ``schema_name`` is a single-schema default. A CSV list or --all-schemas - # has no single value to persist, and persisting the first would silently - # narrow every later bare `slayer ingest`. + # ``schema_name`` persists a single-schema default only; a list or + # --all-schemas has none, and persisting the first would narrow later runs. persisted_schema = ( schemas[0] if schemas and len(schemas) == 1 and not all_schemas else None ) @@ -2043,9 +2040,8 @@ def _run_datasources_create(args, storage): exclude = _parse_csv_arg(args.exclude) try: - # The parsed list is passed explicitly rather than relying on the - # persisted ``schema_name``, so the two can never be read as a - # conflict — ``schema=`` is deliberately never passed on this path. + # Scope passed explicitly, not via the persisted ``schema_name``, so the + # two can't conflict — ``schema=`` is never passed on this path. models = ingest_datasource( datasource=ds, schemas=schemas, diff --git a/slayer/core/enums.py b/slayer/core/enums.py index efa66990..74c7acb8 100644 --- a/slayer/core/enums.py +++ b/slayer/core/enums.py @@ -157,11 +157,9 @@ class JoinType(StrEnum): INNER = "inner" -# The kind of database object a ``sql_table``-mode model points at. -# A plain ``Literal`` rather than a ``StrEnum`` because the values are written -# straight into persisted YAML/SQLite and read back by ``SlayerModel``; keeping -# them bare strings avoids an enum-serialisation round-trip on a field whose -# whole job is to be inspected by humans and agents. +# The kind of database object a ``sql_table``-mode model points at. A bare +# ``Literal`` (not ``StrEnum``): the values are written straight into persisted +# YAML/SQLite, so plain strings avoid an enum-serialisation round-trip. ObjectKind = Literal["table", "view", "materialized_view"] diff --git a/slayer/core/models.py b/slayer/core/models.py index 5ed29efe..f098626e 100644 --- a/slayer/core/models.py +++ b/slayer/core/models.py @@ -124,11 +124,9 @@ def _validate_model_name(name: str, context: str) -> str: def sanitize_model_name(name: str) -> str: """Collapse runs of 2+ underscores so ``name`` passes ``_NO_DUNDER``. - Regex, not ``replace("__", "_")``: ``str.replace`` is non-overlapping, so - ``"a___b"`` would become ``"a__b"`` and still fail validation. - - Only ``__`` is handled — a dotted name would leave ``sql_table`` ambiguous - with schema qualification, so the caller skips those instead. + Regex, not ``replace("__", "_")`` (non-overlapping: ``"a___b"`` → ``"a__b"`` + still fails). Only ``__`` — a dotted name would leave ``sql_table`` + ambiguous with schema qualification, so the caller skips those. """ return _DUNDER_RUN_RE.sub("_", name) @@ -479,12 +477,10 @@ class SlayerModel(BaseModel): version: int = 8 name: str sql_table: str | None = None - # What kind of database object ``sql_table`` names. ``None`` - # means unknown — correct for pre-v8 persisted models, hand-authored - # models, and ``sql`` / query-backed models, which have no live object to - # classify. Only auto-ingestion sets it. Views carry no primary key, so - # this is what explains *why* a model has none and that re-ingesting will - # never produce one. + # What kind of database object ``sql_table`` names. ``None`` = unknown: + # pre-v8, hand-authored, and ``sql``/query-backed models have no live object + # to classify. Only auto-ingestion sets it; it explains why a view-backed + # model has no primary key. source_kind: Optional[ObjectKind] = None sql: str | None = None source_queries: Annotated[ diff --git a/slayer/engine/ingestion.py b/slayer/engine/ingestion.py index 6776544c..1117b017 100644 --- a/slayer/engine/ingestion.py +++ b/slayer/engine/ingestion.py @@ -319,12 +319,9 @@ def _get_fk_relationships( ) -> list[tuple]: """Get FK relationships for a table, filtered to tables in table_set. - Returns list of (source_column, target_table, target_column). - - FK lookup is guarded: views carry no foreign keys and some dialects raise - rather than returning an empty list. This helper feeds ``_build_fk_graph``, - which runs before per-object model construction, so an unguarded raise - would abort the entire ingest. + Returns list of (source_column, target_table, target_column). FK lookup is + guarded: views have no FKs and some dialects raise instead of returning + ``[]``, and this runs before model construction. """ try: fks = inspector.get_foreign_keys(table_name, schema=schema) @@ -461,11 +458,10 @@ def _get_pk_constraint_fallback( ) -> dict: """Get PK constraint via INFORMATION_SCHEMA when Inspector.get_pk_constraint() fails. - On DuckDB this is the path that actually runs — its Inspector reports an - empty ``constrained_columns`` even for a declared PRIMARY KEY — so it has - to understand the same catalog-qualified schema token discovery uses. - ``table_schema`` alone holds the bare name, and filtering on it with a - qualified token silently matched nothing, dropping every primary key. + This is the live path on DuckDB (its Inspector reports empty + ``constrained_columns`` for a declared PK). It splits the catalog-qualified + token because ``table_schema`` holds only the bare name — filtering it with + a qualified token matches nothing and drops every PK. """ if schema: catalog, bare_schema = split_schema_token(schema) @@ -478,10 +474,9 @@ def _get_pk_constraint_fallback( if catalog is not None: clauses.append("tc.table_catalog = :catalog") params["catalog"] = catalog - # The join carries the catalog too: constraint names are only unique - # within a catalog, and DuckDB generates the same ``t_id_pkey`` for a - # same-shaped table in an ATTACHed one — which joined across catalogs - # and returned the PK column twice. + # Join on catalog too: constraint names are unique only within a + # catalog, so an ATTACHed same-shaped table's ``t_id_pkey`` would + # otherwise join across catalogs and duplicate the PK column. sql = ( "SELECT kcu.column_name " "FROM information_schema.table_constraints tc " @@ -752,9 +747,8 @@ def _sqlite_probe_integer_columns( def _parse_qualified_sql_table(sql_table: str) -> tuple[str | None, str]: """Split ``"schema.table"`` into ``(schema, table)`` or ``(None, table)``. - Delegates to :func:`split_sql_table` so a three-part - ``catalog.schema.table`` keeps its catalog instead of losing it to a - split on the first dot. + Delegates to :func:`split_sql_table` so a three-part ``catalog.schema.table`` + keeps its catalog instead of losing it on the first dot. """ return split_sql_table(sql_table) @@ -773,10 +767,8 @@ def introspect_table_to_model( This is the building block shared between the auto-ingest path and the dbt hidden-model import. It never builds joins or traverses the FK graph. - - ``source_kind`` defaults to ``None``: the dbt and OSI converters call this - without classifying the live object, and ``None`` correctly means "not - known" rather than a guess. + ``source_kind`` defaults to ``None`` ("not known"): dbt/OSI callers don't + classify the live object. """ columns = _introspect_query_columns_via_inspector( sa_engine=sa_engine, @@ -808,9 +800,8 @@ def introspect_table_to_model( with warnings.catch_warnings(): - # ``schema`` shadows Pydantic v2's deprecated ``BaseModel.schema()``. - # The name is deliberate — it is the SQLAlchemy ``Inspector`` keyword this - # value is passed as — and the shadowed classmethod is never called here. + # ``schema`` deliberately shadows Pydantic v2's deprecated + # ``BaseModel.schema()`` — it mirrors the SQLAlchemy ``Inspector`` keyword. warnings.filterwarnings("ignore", message='Field name "schema"') class IngestableObject(BaseModel): @@ -818,19 +809,18 @@ class IngestableObject(BaseModel): name: str kind: ObjectKind - # The discovery token the object was found under — catalog-qualified - # on dialects that qualify (DuckDB), bare elsewhere, ``None`` when the - # dialect reports no default schema. This is the string handed back to - # the Inspector; the qualifier written into ``sql_table`` is a - # different string (see :func:`qualify_sql_table`). + # Discovery token the object was found under (handed back to the + # Inspector): catalog-qualified where the dialect qualifies (DuckDB), + # bare elsewhere, ``None`` for no default schema. NOT the qualifier + # written to ``sql_table`` — see :func:`qualify_sql_table`. schema: str | None = None class SkippedTable(BaseModel): """A live object that could not be turned into a model. - Distinct from ``IngestionError`` ("this model failed to persist") — separate - cause, separate fix, so reported separately. + Distinct from ``IngestionError`` (a persist failure) — separate cause and + fix, reported separately. """ table_name: str @@ -843,20 +833,15 @@ class IngestionScanReport(BaseModel): models: list[SlayerModel] = Field(default_factory=list) skipped: list[SkippedTable] = Field(default_factory=list) - # Every object discovered, whether or not it produced a model. Lets the - # caller tell "the schema was empty" apart from "the schema had objects - # but they were all skipped / already in sync" — the CLI needs that - # distinction to decide between the empty-schema hint and silence. + # Every discovered object, whether or not it produced a model — lets the + # CLI tell an empty schema apart from one whose objects were all skipped. objects: list[IngestableObject] = Field(default_factory=list) - # Set when exactly one schema was scanned and the datasource holds - # others: narrowing the default scan is a behaviour change, so it has to - # be visible. Never an error — a hint is not a failure. + # Set when one schema was scanned and others exist: narrowing the default + # scan is a visible behaviour change, not an error. schema_hint: str | None = None - # Object names living in the connection's default schema. The additive - # pass needs it to tell "this persisted unqualified model IS the default - # schema's table" from "a same-named table in another schema", which is - # the one case a schema comparison alone cannot decide. ``None`` means the - # listing failed — distinct from empty, because the consumer fails closed. + # Object names in the default schema, for the additive pass to tell "this + # unqualified model IS the default schema's table" from a same-named table + # elsewhere. ``None`` (listing failed) is distinct from empty: fails closed. default_schema_objects: list[str] | None = None @@ -877,9 +862,8 @@ class IngestionScanReport(BaseModel): } ) _SYSTEM_SCHEMA_PREFIXES = ("pg_temp_", "pg_toast_temp_") -# DuckDB exposes its own metadata and scratch space as first-segment -# catalogs (``system.main``, ``temp.main``), so those are matched on the -# catalog rather than on the schema name. +# DuckDB exposes metadata/scratch space as first-segment catalogs +# (``system.main``, ``temp.main``), matched on catalog not schema name. _SYSTEM_CATALOGS = frozenset({"system", "temp"}) @@ -895,18 +879,13 @@ def _is_system_schema(token: str) -> bool: class ResolvedSchema(BaseModel): """One schema in ingest scope. - ``name`` is the *discovery* token, in the shape the dialect enumerates — - catalog-qualified on DuckDB. A schema the user named bare is upgraded to - that shape, because a bare token reaches into ``ATTACH``ed catalogs. - - ``requested_as`` is what the user actually typed, and is what gets written - into ``sql_table`` on the ``explicit`` path — resolving a token for - discovery must never change the SQL we emit. - - ``explicit`` means the user named this single schema, so its qualifier is - written verbatim; a multi-schema request follows the automatic rules - instead, or listing the default schema alongside another would re-qualify - every model already on disk. + ``name`` is the *discovery* token (catalog-qualified on DuckDB); a bare + user-named schema is upgraded to that shape, since a bare token reaches into + ``ATTACH``ed catalogs. ``requested_as`` keeps what the user typed and is what + the ``explicit`` path writes to ``sql_table``, so discovery upgrades never + change emitted SQL. ``explicit`` (user named this single schema) writes the + qualifier verbatim; a multi-schema request uses the automatic rules instead, + else the default schema would get re-qualified across every model on disk. """ name: str | None = None @@ -935,10 +914,9 @@ def _bare_schema(token: str | None) -> str: def _matches_default(token: str, default_token: str | None) -> bool: """Whether a user-supplied token names the connection's default schema. - A qualified token must match in full — that is what stops ``aaa.main`` - and ``att_main.main`` both reading as the default. A bare token is - compared against the default's bare segment, so ``--schema main`` on - DuckDB still recognises ``fda.main``. + A qualified token must match in full (so ``aaa.main`` and ``att_main.main`` + don't both read as default); a bare token matches the default's bare segment, + so ``--schema main`` still recognises ``fda.main`` on DuckDB. """ if not default_token: return False @@ -954,10 +932,9 @@ def _current_catalog_only( ) -> tuple[list[str], list[SkippedTable]]: """Restrict enumerated tokens to the connection's current catalog. - ``--all-schemas`` means "this database", not "and everything anyone has - attached to the session" — DuckDB's ``get_schema_names()`` lists - ``ATTACH``ed catalogs too. Dropped tokens are reported with the exact - invocation that would ingest them. + ``--all-schemas`` means "this database", not the ``ATTACH``ed catalogs + DuckDB's ``get_schema_names()`` also lists. Dropped tokens are reported + with the invocation that would ingest them. """ if not any("." in t for t in tokens): return list(tokens), [] @@ -1002,10 +979,9 @@ def _resolved_from_request( ) -> ResolvedSchema: """Build a :class:`ResolvedSchema` for a schema the user named. - The discovery token is upgraded to the enumerated (catalog-qualified) - shape where that is unambiguous — a bare ``main`` otherwise sweeps - ``ATTACH``ed catalogs — while ``requested_as`` keeps the user's own string - so the emitted ``sql_table`` is unchanged by the upgrade. + The discovery token is upgraded to the catalog-qualified shape where + unambiguous (a bare ``main`` otherwise sweeps ``ATTACH``ed catalogs); + ``requested_as`` keeps the user's string so ``sql_table`` is unchanged. """ return ResolvedSchema( name=resolve_schema_token(inspector, token, enumerated=enumerated), @@ -1026,8 +1002,7 @@ def resolve_ingest_schemas( Precedence, first non-empty wins: ``all_schemas``; an explicit ``requested`` list; the datasource's persisted ``schema_name``; the - connection's default schema. The persisted value is a *fallback*, - consulted only when nothing more specific was given — never a conflict. + connection's default schema. """ default_token = qualified_default_schema(inspector) enumerated = _enumerate_schemas(inspector) @@ -1043,9 +1018,9 @@ def resolve_ingest_schemas( for token in local ] elif requested: - # A single named schema is honoured verbatim (``--schema public`` - # keeps producing ``public.orders``). A list is not: qualifying the - # default schema verbatim there would rewrite every model on disk. + # A single named schema is honoured verbatim (``--schema public`` keeps + # producing ``public.orders``); a list is not, else the default schema + # would get re-qualified across every model on disk. single = len(requested) == 1 schemas = [ _resolved_from_request( @@ -1082,18 +1057,13 @@ def resolve_ingest_schemas( def qualify_sql_table(*, obj: IngestableObject, resolved: ResolvedSchema) -> str: - """The ``sql_table`` value for ``obj``. - - Note this is NOT the discovery token: the catalog segment is dropped - because the connection's current catalog is already the right one, and - re-stating it would only break if the datasource were repointed. - - Default-schema objects stay unqualified so that widening the scan never - rewrites models that already exist. + """The ``sql_table`` value for ``obj`` — NOT the discovery token. - The explicit path emits ``requested_as`` — what the user typed — not the - resolved discovery token, so upgrading ``main`` to ``fda.main`` for safe - introspection cannot leak a catalog into the persisted SQL. + The catalog segment is dropped (the current catalog is already right). + Default-schema objects stay unqualified so widening the scan never rewrites + existing models. The explicit path emits ``requested_as`` (what the user + typed), so an introspection upgrade like ``main`` → ``fda.main`` can't leak + a catalog into persisted SQL. """ if resolved.explicit: return f"{resolved.requested_as or resolved.name}.{obj.name}" @@ -1108,12 +1078,11 @@ def _safe_object_names( inspector: sa.engine.Inspector, schema: str | None, ) -> list[str]: - """Call an ``Inspector.get_*_names`` accessor, tolerating dialects that - lack it. + """Call an ``Inspector.get_*_names`` accessor, tolerating dialects that lack it. ``get_materialized_view_names`` raises ``NotImplementedError`` where - unsupported, so it cannot be called bare. Driver errors are tolerated too — - broken view discovery must not stop tables ingesting. + unsupported; driver errors are swallowed too, so broken view discovery + can't stop tables ingesting. """ accessor = getattr(inspector, accessor_name, None) if accessor is None: @@ -1136,15 +1105,12 @@ def list_ingestable_objects( ) -> list[IngestableObject]: """Discover every ingestable object in ``schema``, classified by kind. - ``schema=None`` resolves to the connection's *default schema token* - rather than being passed through. On DuckDB a bare ``None`` returns - objects from every schema as bare names, which is how a model in a - non-default schema ended up with an unqualified ``sql_table``; and even - the bare default (``main``) still reaches into ``ATTACH``ed catalogs, - so the token has to carry the catalog. - - Order is deterministic (tables, views, matviews). Deduped across - accessors — some dialects return views from ``get_table_names()``. + ``schema=None`` resolves to the default schema *token* rather than passing + through: on DuckDB a bare ``None`` returns every schema's objects as bare + names (how a non-default model lost its ``sql_table`` qualifier), and even + the bare default reaches into ``ATTACH``ed catalogs, so the token carries + the catalog. Order is deterministic (tables, views, matviews); deduped + across accessors, since some dialects return views from ``get_table_names()``. """ resolved_schema = ( schema if schema is not None else qualified_default_schema(inspector) @@ -1224,19 +1190,13 @@ def _assign_model_names( ) -> tuple[dict[tuple[str | None, str], str], list[SkippedTable]]: """Map each ``(schema, object name)`` to its model name. - Returns ``(mapping, skipped)``. Model names may not contain ``__`` (the - SQL generator splits it back into a join path, so ``a__b`` would silently - query ``a -> b``); object names may, so only the model name is sanitized. - - Contention is resolved in ONE phase keyed on the final model name, not as - successive passes: two objects can now compete either because one was - sanitized into the other's name or because they live in different schemas, - and resolving those separately would leave the mixed case (``s1.a__b`` - sanitizing onto a real ``s2.a_b``) dependent on reservation order. + Returns ``(mapping, skipped)``. Model names may not contain ``__`` (the SQL + generator splits it into a join path, so ``a__b`` would query ``a -> b``); + object names may, so only the model name is sanitized. - The winner is fixed by a total order over the contenders, every key of - which is a property of the object itself — so the outcome is independent - of inspector listing order and of ``--schema`` argument order: + Contention (two objects competing for one model name, via sanitization or + different schemas) is resolved in one phase by a total order over the + contenders, so the winner is independent of listing and ``--schema`` order: 1. a name needing no sanitization beats one that did; 2. then the default schema beats a non-default one; @@ -1303,12 +1263,9 @@ def _build_one_model( fk_columns_by_table: dict[str, set[str]], table_set: set[str], ) -> SlayerModel: - """Introspect one live object into a model. Raises on failure; the caller - isolates per-object. - - Introspection is driven by the *discovery* token (``resolved.name``) while - the emitted ``sql_table`` carries the qualifier — two different strings - that must not be conflated. + """Introspect one live object into a model. Raises on failure; caller + isolates per-object. Introspection uses the discovery token + (``resolved.name``); the emitted ``sql_table`` carries the qualifier. """ referenced = ( set() if has_cycles else _compute_transitive_closure(fk_graph, obj.name) @@ -1354,13 +1311,9 @@ def _build_one_model( def _dispose_quietly(sa_engine: sa.Engine) -> None: """Dispose ``sa_engine``, logging rather than raising on failure. - Called from ``finally`` blocks, so raising would replace the in-flight - exception (or turn a successful run into a failure) — the caller would see - a teardown error instead of the driver error that actually failed. - - Logged at WARNING, not DEBUG: disposal is what releases the connection so - an external ``duckdb.connect(file)`` can open the same file, so a failure - here is a real resource leak and needs to be operationally visible. + Called from ``finally``, so raising would mask the in-flight driver error. + Logged at WARNING because disposal releases the connection (so an external + ``duckdb.connect(file)`` can reopen the file); a failure here leaks it. """ try: sa_engine.dispose() @@ -1378,9 +1331,8 @@ def _collect_fk_columns( ) -> dict[str, set[str]]: """Map each table to its FK-constrained columns, for rollup exclusion. - Guarded per table: views have no foreign keys and some dialects raise - instead of returning ``[]``. This runs before per-object model - construction, so an unguarded raise would abort the whole ingest. + Guarded per table: views have no FKs and some dialects raise instead of + returning ``[]``, and this runs before model construction. """ out: dict[str, set[str]] = defaultdict(set) for table_name in table_names: @@ -1398,10 +1350,9 @@ def _collect_fk_columns( def _schema_hint(scope: IngestSchemaScope) -> str | None: """Tell the user which schemas the scan left out, and how to get them. - Eligibility is "exactly one schema in scope and others exist" — it is - deliberately independent of whether that schema was named explicitly. A - user who set ``schema_name`` months ago still needs to hear that another - schema has appeared since. + Eligibility ("one schema in scope and others exist") is independent of + whether that schema was named explicitly — a datasource that gained a + schema since ``schema_name`` was set still needs the hint. """ if len(scope.schemas) != 1 or not scope.other_schemas: return None @@ -1425,7 +1376,7 @@ def _scan_one_schema( multi_schema: bool, ) -> tuple[list[SlayerModel], list[SkippedTable]]: """Build every model for one schema. The FK graph is per-schema: joins - only ever resolve within the schema the objects were discovered in.""" + resolve only within the schema the objects were discovered in.""" table_names = [o.name for o in objects] table_set = set(table_names) schema = resolved.name @@ -1516,15 +1467,11 @@ def _default_schema_object_names( ) -> list[str] | None: """Object names living in the connection's default schema. - Derived from the objects already discovered when the default schema is in - scope; otherwise listed explicitly, which costs one extra catalog call on - the only path that needs it. - - ``None`` means "could not be determined" and is deliberately distinct from - the empty list. The consumer is a guard against repointing a model at a - different physical table, so an unknown answer has to fail CLOSED — an - empty list would read as "no default-schema object of that name exists" - and wave the repoint through. + Derived from already-discovered objects when the default schema is in + scope, else listed explicitly (one extra catalog call). ``None`` ("could + not be determined") is distinct from empty: the consumer guards against + repointing a model, so an unknown answer fails CLOSED where an empty list + would wave the repoint through. """ default_token = qualified_default_schema(inspector) if any(s.name == default_token for s in scope.schemas): @@ -1625,13 +1572,9 @@ def ingest_datasource_report( ), ) finally: - # One-shot admin operation, not a hot query path. Disposing releases - # the underlying connection so other consumers (notably - # ``duckdb.connect(file)`` in notebooks) can open the same file. In a - # ``finally`` because discovery, the FK graph, and the FK-column pass - # can all raise a driver error — the REST layer explicitly handles - # ``SQLAlchemyError`` from here — and an undisposed engine would keep - # the connection open, which is the exact problem this prevents. + # In ``finally`` because discovery and the FK passes can raise a driver + # error (the REST layer handles ``SQLAlchemyError`` from here); see + # ``_dispose_quietly`` for why disposal matters. _dispose_quietly(sa_engine) @@ -1792,16 +1735,11 @@ def _additive_merge_existing( * Live columns whose names are absent from ``persisted.columns`` are appended from ``fresh.columns``. * Joins with new ``(target_model, join_pairs)`` signatures are appended. - * Carve-out: ``source_kind`` is REFRESHED, not preserved. It - describes the live object, not user intent, and the transition it exists - to capture — dbt's ``+materialized: table`` turning a view into a table — - usually changes no columns at all. A field that never refreshed would - confidently lie about precisely the case it was added for. A ``None`` - from a path that doesn't classify never erases a known value. - * Carve-out: a MISSING ``sql_table`` schema qualifier is repaired, so a - model ingested before schemas were recorded becomes queryable again on - the next run. An existing qualifier is never rewritten — healing adds - one, it does not repoint a model someone deliberately aimed elsewhere. + * Carve-out: ``source_kind`` is REFRESHED, not preserved — it describes the + live object, and a view→table flip (dbt ``+materialized: table``) usually + changes no columns. A ``None`` from a non-classifying path never erases it. + * Carve-out: a MISSING ``sql_table`` qualifier is repaired so a pre-schema + model is queryable again; an existing qualifier is never rewritten. """ existing_by_name: dict[str, Column] = {c.name: c for c in persisted.columns} fresh_by_name: dict[str, Column] = {c.name: c for c in fresh.columns} @@ -1828,17 +1766,14 @@ def _additive_merge_existing( new_joins, new_join_targets = _merge_joins_strict(persisted, fresh) - # A view→table flip typically changes nothing else, so the kind check has - # to participate in the short-circuit below — not just in the update dict — - # or the refresh would never be reached. + # Gates the short-circuit below (not just the update dict), else a + # column-less view→table flip would never reach the refresh. kind_changed = ( fresh.source_kind is not None and fresh.source_kind != persisted.source_kind ) - # A qualifier repair typically changes nothing else either, so it has to - # participate in the short-circuit for the same reason ``kind_changed`` - # does — otherwise the repaired model is computed and then discarded. + # Gates the short-circuit too, for the same reason as ``kind_changed``. sql_table_change = _qualifier_repair(persisted=persisted, fresh=fresh) if not ( @@ -1872,8 +1807,8 @@ def _qualifier_repair( """``"before → after"`` when ``fresh`` supplies a qualifier ``persisted`` is missing, else ``None``. - Keyed on the bare object names matching: ``reports`` and ``s.other`` are - unrelated tables that happen to share a model name, not a repair. + Requires the bare names to match, so two unrelated tables sharing a model + name aren't mistaken for a repair. """ before, after = persisted.sql_table, fresh.sql_table if not before or not after: @@ -1888,8 +1823,7 @@ def _qualifier_repair( class ProcessTableOutcome(BaseModel): """What the additive pass did with one freshly-introspected model. - A skip ("I declined this one") is not an error ("this failed to persist"), - so it travels separately and is reported separately. + A skip (declined) is not an error (persist failure), so travels separately. """ addition: Any | None = None @@ -1905,17 +1839,15 @@ def _cross_schema_conflict( ) -> SkippedTable | None: """Refuse to merge two different schemas' tables into one model. - Two sequential single-schema ingests would otherwise fuse them, with no - flag involved. The second check covers the case a schema comparison alone - cannot: a default-schema model is persisted *unqualified*, so a fresh - qualified object with the same bare name looks like a repair when it is - actually a different table. - - ``default_schema_objects=None`` means the default schema could not be - listed. That fails CLOSED: an unqualified persisted model is treated as a - possible default-schema table and the merge is refused. Skipping a legal - repair costs a re-run; guessing wrong repoints a model at another - schema's data. + Two sequential single-schema ingests would otherwise fuse them. The second + check covers what a schema comparison cannot: a default-schema model is + persisted *unqualified*, so a fresh qualified object with the same bare name + looks like a repair but is a different table. + + ``default_schema_objects=None`` (default schema unlistable) fails CLOSED — + the unqualified persisted model is treated as possibly default-schema and + the merge refused. Skipping a legal repair costs a re-run; guessing wrong + repoints a model at another schema's data. """ persisted_table = persisted.sql_table or "" fresh_table = fresh.sql_table or "" @@ -1986,10 +1918,8 @@ async def _process_one_table( fresh=fresh, sqlite_widen_enabled=(datasource.type or "").lower() == "sqlite", ) - # ``kind_changed`` and ``sql_table_change`` must gate the save too: a - # view→table flip or a qualifier repair usually adds no columns and no - # joins, so without them the refreshed model would be computed and then - # thrown away. + # ``kind_changed`` / ``sql_table_change`` gate the save too, else a + # column-less view→table flip or qualifier repair would be dropped. if ( outcome.new_columns or outcome.new_joins @@ -2025,8 +1955,7 @@ def _bare_table_name(sql_table: str) -> str: def _schema_of(sql_table: str) -> str | None: """The bare schema segment of a ``sql_table``, or None when unqualified. - Built on :func:`split_sql_table` so there is one dotted-name parser, not - two that disagree about three-part names. + Built on :func:`split_sql_table` so there is one dotted-name parser. """ schema_token, _ = split_sql_table(sql_table) return _bare_schema(schema_token) if schema_token else None @@ -2120,11 +2049,9 @@ async def ingest_datasource_idempotent( else set(scan.default_schema_objects) ) fresh_by_name = {m.name: m for m in fresh_models} - # Keyed on the LIVE OBJECT name, not the model name. ``_scoped_models_for_validation`` - # compares this against ``_bare_table_name(m.sql_table)``, so using model - # names silently dropped from validation scope any model whose name differs - # from its table — every ``__``-sanitized model, and already - # every dbt/OSI hidden model that passes ``model_name=``. + # Keyed on the live object name (via ``sql_table``), not the model name: + # ``_scoped_models_for_validation`` matches ``_bare_table_name(sql_table)``, + # so model names would drop every renamed/sanitized model from scope. in_scope_table_names: set[str] = { _bare_table_name(m.sql_table) for m in fresh_models if m.sql_table } @@ -2289,8 +2216,7 @@ def _print_ingest_addition( return widened = getattr(addition, "widened_columns", []) or [] kind_change = getattr(addition, "kind_change", None) - # A qualifier repair adds no columns, so without it in the gate the whole - # line — the point of the re-ingest — would print nothing at all. + # A qualifier repair adds no columns, so it must gate this line too. sql_table_change = getattr(addition, "sql_table_change", None) if not ( addition.new_columns @@ -2322,9 +2248,8 @@ def _print_ingest_drift_and_errors( print("\nPending drift (run `slayer validate-models` to inspect):", file=out) for entry in result.to_delete: print(f" - {entry.tool}: {entry.model_name}", file=out) - # Skips are reported separately from errors — "this object can't - # be modelled" has a different cause and a different fix from "this model - # failed to persist". + # Skips (not modellable) are reported apart from errors (persist failures) — + # different cause, different fix. skipped = getattr(result, "skipped", None) or [] if skipped: print( diff --git a/slayer/engine/introspect_utils.py b/slayer/engine/introspect_utils.py index 967dbb1d..688638c3 100644 --- a/slayer/engine/introspect_utils.py +++ b/slayer/engine/introspect_utils.py @@ -74,10 +74,8 @@ def _parse_info_schema_is_float(data_type_str: str) -> bool: def split_sql_table(sql_table: str) -> tuple[Optional[str], str]: """Split a ``sql_table`` reference into ``(schema_token, object_name)``. - The schema token is everything before the FINAL dot, catalog segment - included — ``proj.dataset.tbl`` yields ``("proj.dataset", "tbl")``. - Truncating it would introspect the wrong catalog, or (on DuckDB) match - nothing at all. + Splits on the FINAL dot, keeping the catalog: ``proj.dataset.tbl`` → + ``("proj.dataset", "tbl")``. Truncating would introspect the wrong catalog. """ schema_token, sep, obj = sql_table.rpartition(".") if not sep: @@ -130,17 +128,13 @@ def enumerated_schema_names(inspector: sa.engine.Inspector) -> list[str]: def _enumerates_qualified_tokens(inspector: sa.engine.Inspector) -> bool: """Whether this dialect prefixes its schema tokens with a catalog. - Decided by asking where the dialect's OWN default schema turns up in its - OWN enumeration: listed bare means bare tokens (SQLite, Postgres, MySQL, - BigQuery); absent but present as some ``.`` means the - dialect qualifies (DuckDB). - - Deliberately not "does any enumerated name contain a dot" — Postgres - allows ``CREATE SCHEMA "foo.bar"``, and reading that as a catalog would - let it capture a request for a nonexistent ``bar``. Deliberately not - ``qualified_default_schema()`` either: that helper falls back to the bare - default when the current catalog cannot be determined, which would - misreport DuckDB as bare and re-arm the cross-catalog sweep. + Decided by where the dialect's own default schema appears in its own + enumeration: listed bare → bare tokens (SQLite/Postgres/MySQL/BigQuery); + absent but present as ``.`` → qualifies (DuckDB). + + Not "any enumerated name contains a dot" (Postgres allows + ``CREATE SCHEMA "foo.bar"``), and not ``qualified_default_schema()`` (its + bare fallback would misreport DuckDB and re-arm the cross-catalog sweep). """ try: default = inspector.default_schema_name @@ -163,16 +157,11 @@ def resolve_schema_token( ) -> Optional[str]: """Resolve a user-supplied schema name to a safe *discovery* token. - A user types ``--schema main``, and a persisted ``sql_table`` records the - bare ``analytics``. On DuckDB the bare form is the dangerous one — it makes - ``get_table_names`` and ``has_table`` reach into ``ATTACH``ed catalogs — so - a bare token is upgraded to the catalog-qualified token the dialect - enumerates, when exactly one matches. Ambiguous or unknown names are - returned unchanged: guessing between two catalogs would be worse than - letting the accessors report nothing. - - Dialects that enumerate bare names resolve to themselves, so this is a - no-op everywhere except DuckDB. + On DuckDB a bare token (``--schema main``, or a persisted bare ``sql_table``) + makes ``get_table_names`` / ``has_table`` reach into ``ATTACH``ed catalogs, + so it is upgraded to the catalog-qualified token the dialect enumerates when + exactly one matches. Ambiguous or unknown names are returned unchanged. A + no-op on dialects that enumerate bare names. """ if not token: return token @@ -180,23 +169,19 @@ def resolve_schema_token( if not names or token in names: return token if "." in token: - # Already carries a catalog and is not a known schema — honour it - # verbatim rather than second-guessing the user. + # Already carries a catalog but isn't a known schema — honour verbatim. return token if not _enumerates_qualified_tokens(inspector): - # This dialect lists bare schema names, so a dot in one is part of the - # name (Postgres allows `CREATE SCHEMA "foo.bar"`), not a catalog - # separator. Reading it as one would silently ingest a schema the user - # did not ask for; an unknown name should simply find nothing. + # This dialect lists bare names, so a dot is part of the name (Postgres + # ``CREATE SCHEMA "foo.bar"``), not a catalog separator. return token matches = [n for n in names if n.rsplit(".", 1)[-1] == token] if len(matches) == 1: return matches[0] if not matches: return token - # Several catalogs expose a schema of this name. A bare name means the - # one in the database we are connected to — the same thing an unqualified - # ``FROM main.t`` resolves to — so prefer it over guessing or giving up. + # Several catalogs expose this schema name; a bare name means the one in + # the connected database, as unqualified ``FROM main.t`` resolves. catalog = _current_catalog(inspector) if catalog: in_catalog = [n for n in matches if n.split(".", 1)[0] == catalog] @@ -210,15 +195,11 @@ def qualified_default_schema( ) -> Optional[str]: """The discovery token identifying the connection's default schema. - DuckDB's ``get_schema_names()`` always returns catalog-qualified tokens - (``fda.main``), and the qualified form is the *safe* one: a bare ``main`` - makes ``get_table_names`` and ``has_table`` reach into ``ATTACH``ed - catalogs. So the token is matched against what the dialect actually - enumerates, and returned in exactly that shape. Dialects that enumerate - bare names get their bare default back unchanged. - - Returns ``None`` when the dialect reports no default schema, which callers - treat as "pass ``None`` to the accessors", i.e. today's behaviour. + Returned in the shape the dialect enumerates: DuckDB qualifies (``fda.main``), + and the qualified form is the *safe* one (a bare ``main`` reaches into + ``ATTACH``ed catalogs). Bare-enumerating dialects get their bare default + back. ``None`` when the dialect reports no default schema (callers pass + ``None`` to the accessors). """ cached = getattr(inspector, _DEFAULT_SCHEMA_ATTR, _UNSET) if cached is not _UNSET: @@ -250,8 +231,8 @@ def _compute_default_schema_token( catalog = _current_catalog(inspector) if catalog and f"{catalog}.{default}" in names: return f"{catalog}.{default}" - # A dialect that qualifies its tokens but whose current catalog we could - # not determine: accept a unique last-segment match, never an ambiguous one. + # Qualifying dialect, current catalog unknown: accept a unique last-segment + # match, never an ambiguous one. matches = [n for n in names if n.rsplit(".", 1)[-1] == default] return matches[0] if len(matches) == 1 else default @@ -283,11 +264,10 @@ def _columns_in_schema( ) -> List[Dict]: """Columns of ``schema.table_name``, filtered on the catalog too. - ``information_schema.columns`` carries ``table_catalog``, and it is - populated for ``ATTACH``ed catalogs — so a catalog-qualified token filters - exactly, instead of matching nothing (which silently produced a - column-less model) or matching everywhere (which unioned two tables' - columns together). + ``information_schema.columns.table_catalog`` is populated for ``ATTACH``ed + catalogs, so a catalog-qualified token filters exactly — otherwise it + matches nothing (column-less model) or everywhere (two tables' columns + unioned). """ catalog, bare_schema = split_schema_token(schema) clauses = ["table_name = :table_name", "table_schema = :schema"] @@ -315,16 +295,13 @@ def _get_columns_fallback( ) -> List[Dict]: """Get columns via INFORMATION_SCHEMA when Inspector.get_columns() fails. - On DuckDB this is not a rare backstop — ``Inspector.get_columns`` raises - for every schema — so it is the primary column path for a Tier-1 dialect. + On DuckDB this is the primary column path, not a rare backstop — + ``Inspector.get_columns`` raises for every schema. - With no ``schema`` the query cannot be narrowed, so same-named tables in - two schemas both match. Rather than unioning their columns (which produced - models referencing columns their table does not have) the rows are grouped - by catalog+schema: one group is used, ``default_schema`` breaks a tie, and - anything still ambiguous raises. Picking a winner by sort order would - replace union corruption with wrong-table corruption, which is harder to - notice; the caller isolates the raise per object and reports a skip. + With no ``schema`` the query can't be narrowed, so same-named tables in two + schemas both match. Instead of unioning their columns, rows are grouped by + catalog+schema: a single group is used, ``default_schema`` breaks a tie, and + anything still ambiguous raises (the caller isolates it per object as a skip). """ if schema: return _columns_in_schema(sa_engine, table_name, schema) diff --git a/slayer/engine/schema_drift.py b/slayer/engine/schema_drift.py index 0aafa1ca..6f9c1164 100644 --- a/slayer/engine/schema_drift.py +++ b/slayer/engine/schema_drift.py @@ -111,15 +111,14 @@ class ModelAddition(BaseModel): # DEV-1538: persisted INT columns whose type widened (to DOUBLE or TEXT) # because the SQLite affinity probe disagreed with the declared type. widened_columns: list[str] = Field(default_factory=list) - # Output metadata — lets the renderer label a view-backed model - # without reloading it. Distinct from the persisted - # ``SlayerModel.source_kind``, which is the durable record. + # Renderer metadata (label a view-backed model without reloading it); + # distinct from the durable ``SlayerModel.source_kind``. source_kind: str | None = None - # Human-readable transition (e.g. "view → table") when a re-ingest found - # the live object had changed kind. None when nothing changed. + # Transition (e.g. "view → table") when a re-ingest found the live object + # changed kind, else None. kind_change: str | None = None - # Human-readable transition (e.g. "reports → openfda_rest.reports") when a - # re-ingest repaired a missing schema qualifier. None when nothing changed. + # Transition (e.g. "reports → openfda_rest.reports") when a re-ingest + # repaired a missing schema qualifier, else None. sql_table_change: str | None = None @@ -137,19 +136,15 @@ class IdempotentIngestResult(BaseModel): additions: list[ModelAddition] = Field(default_factory=list) to_delete: list[ToDeleteEntry] = Field(default_factory=list) errors: list[IngestionError] = Field(default_factory=list) - # ``skipped`` holds live objects that could not be modelled at - # all — reported separately from ``errors`` because the cause and the fix - # differ ("can't represent this name" vs "couldn't persist this model"). + # Unmodellable live objects, reported apart from ``errors`` (different cause + # and fix: "can't represent this name" vs "couldn't persist this model"). skipped: list[Any] = Field(default_factory=list) - # Every live object discovered this pass, whether or not it produced a - # model. Lets the CLI distinguish an empty schema (worth a hint) from a - # healthy no-op re-ingest (worth silence). Typed ``Any`` to avoid a - # circular import with ``engine.ingestion``; runtime entries are - # ``IngestableObject``. + # Every live object discovered this pass, to distinguish an empty schema + # (worth a hint) from a healthy no-op re-ingest. ``Any`` to dodge a circular + # import with ``engine.ingestion``; runtime entries are ``IngestableObject``. objects: list[Any] = Field(default_factory=list) - # Set when exactly one schema was scanned and the datasource holds others. - # Travels in the response body so REST / MCP callers see the same nudge - # the CLI prints. Advisory — never an error. + # Set when one schema was scanned and the datasource holds others; carried + # in the response so REST/MCP see the same nudge the CLI prints. Advisory. schema_hint: str | None = None @@ -1666,11 +1661,10 @@ def compute_datasource_drops( def _alias_keys(schema_token: str | None, name: str) -> list[str]: """Progressively shorter lookup keys for one live object, longest first. - A model written before the catalog was known says ``schema.table``; one - written before schemas were recorded at all says just ``table``. Both must - keep resolving, so both aliases are offered — but a contested alias is - only inserted for the entry the database itself would pick (see - :func:`_index_live_entries`). + A model written before the catalog was known says ``schema.table``; an even + older one says just ``table``. Both must keep resolving, so both aliases are + offered — but a contested one is inserted only for the entry the database + itself would pick (see :func:`_index_live_entries`). """ if not schema_token: return [name] @@ -1684,25 +1678,20 @@ def _index_live_entries( ) -> dict[str, LiveTable]: """Key the live objects on their full identity, plus shorter aliases. - Full ``"."`` keys are always present. A shorter - alias is inserted when exactly one object claims it, and — when several - do — for the one in the connection's DEFAULT schema, because that is - precisely how the database resolves the short form: ``FROM orders`` and - ``FROM main.orders`` both land in the current catalog's default schema. - - Dropping a contested alias outright looked safer but is not: an - unqualified ``sql_table`` is how every default-schema model is persisted, - so the moment another schema happened to hold a same-named table the - legacy model stopped resolving — and an unresolvable model is a - ``WholeModelDelete`` that ``validate-models --force-clean`` deletes. The - alias is only dropped when the default schema cannot break the tie, where - a miss really is better than an arbitrary winner. + Full ``"."`` keys are always present. A shorter alias + is inserted when exactly one object claims it, or — when several do — for + the one in the DEFAULT schema, since that is how the database resolves the + short form (``FROM orders`` == ``FROM main.orders``). + + A contested alias is dropped only when the default can't break the tie: + default-schema models persist unqualified, so dropping it wholesale would + stop a legacy model resolving the moment another schema held a same-named + table — and an unresolvable model is a ``WholeModelDelete`` that + ``validate-models --force-clean`` deletes. """ - # Deduplicate on identity first. Two requested schemas can resolve to the - # same discovery token (``None`` and ``main`` both become ``fda.main``), - # and a repeat would otherwise look like two rival claimants for every - # alias — silently dropping the aliases of objects that have no rival at - # all, which is the very deletion path this indexing exists to prevent. + # Dedupe on identity first: two requested schemas can resolve to the same + # token (``None`` and ``main`` → ``fda.main``), and a repeat would look like + # two rival claimants, dropping aliases of objects that have no real rival. unique: dict[tuple[str | None, str], LiveTable] = {} for schema_token, name, live in entries: unique.setdefault((schema_token, name), live) @@ -1734,29 +1723,18 @@ def _live_schema_for_datasource( schemas: list[str | None] | None = None, ) -> dict[str, LiveTable]: """Return ``{object_key: LiveTable}`` for every live table AND view in the - listed schemas, using SQLAlchemy ``Inspector`` and the same fallback path - as auto-ingestion (``slayer/engine/ingestion.py``). - - Keys are the FULL discovery identity ``"."``, - catalog segment included, plus shorter aliases inserted only when unique - across everything scanned. On a clash the ambiguous alias is dropped, so - a lookup misses rather than silently resolving to another catalog's - same-named table. - - ``schemas=None`` means the connection's default schema, matching the - previous single-schema signature. - - Views are included **unconditionally** — there is deliberately no - ``include_views`` parameter here, and adding one would be a bug. - - This map is only ever a lookup target: ``validate_datasource`` iterates the - *persisted* models and resolves each model's ``sql_table`` against it, so - including views can never manufacture a model or a drift entry. What it - does fix is the reverse: a model whose ``sql_table`` names a view used to - resolve to ``None``, which ``diff_sql_table_model`` reports as a - ``WholeModelDelete`` — and ``validate-models --force-clean`` acts on that. - Gating this on the ingest-side ``--no-views`` flag would re-arm that - data-loss bug for anyone who opted out of ingesting views. + listed schemas, via the same ``Inspector`` fallback path as auto-ingestion. + + Keys are the full discovery identity ``"."`` plus + shorter aliases inserted only when unique (see :func:`_index_live_entries`). + ``schemas=None`` means the connection's default schema. + + Views are included **unconditionally** (no ``include_views`` param, by + design): this map is only a lookup target, so a view can't manufacture a + model. It fixes the reverse — a model whose ``sql_table`` names a view would + otherwise resolve to ``None``, i.e. a ``WholeModelDelete`` that + ``validate-models --force-clean`` acts on. Gating on ``--no-views`` would + re-arm that data-loss bug. """ from slayer.engine.ingestion import _dispose_quietly, list_ingestable_objects from slayer.engine.introspect_utils import ( @@ -1768,18 +1746,15 @@ def _live_schema_for_datasource( try: inspector = sa.inspect(sa_engine) entries: list[tuple[str | None, str, LiveTable]] = [] - # The schema set is derived from persisted ``sql_table`` values, which - # carry the BARE schema — and a bare token reaches into ATTACHed - # catalogs. Upgrade each to the enumerated shape, then dedupe: `None` - # and `main` both resolve to `fda.main`, and scanning it twice would - # double every entry. + # Schemas come from persisted ``sql_table`` values (bare, so they reach + # into ATTACHed catalogs). Upgrade each to the enumerated shape and + # dedupe: ``None`` and ``main`` both resolve to ``fda.main``, and + # scanning it twice would double every entry. scanned: list[str | None] = [] default_token = qualified_default_schema(inspector) for requested in schemas if schemas is not None else [None]: - # ``None`` means the default schema, which is what - # ``list_ingestable_objects`` resolves it to anyway — normalise it - # here so it dedupes against an explicit request for that same - # schema instead of introspecting everything twice. + # Normalise ``None`` to the default token here so it dedupes against + # an explicit request for the same schema. schema = ( default_token if requested is None else resolve_schema_token(inspector, requested) @@ -1813,11 +1788,9 @@ def _live_schema_for_datasource( entries, default_token=qualified_default_schema(inspector), ) finally: - # Same rationale as ``ingest_datasource``: this is a one-shot - # admin path. Disposing releases the underlying connection so - # external direct file access (e.g. ``duckdb.connect(file)``) - # in the same process isn't blocked. Quiet, so a raising dispose - # can't replace an in-flight introspection error. + # See ``_dispose_quietly``: releases the connection (so external + # ``duckdb.connect(file)`` isn't blocked) without a raising dispose + # masking an in-flight introspection error. _dispose_quietly(sa_engine) @@ -1933,11 +1906,10 @@ def _resolve_live_table( progressively shorter keys and unquoting double-quoted identifiers (e.g. ``prod."Company"`` for case-sensitive Postgres tables). - Order matters: full identity, then the last two segments, then the bare - name. The live map drops ambiguous short keys, so a miss on a shorter - candidate means "this could be either table" — and returning None there - is correct, where returning an arbitrary match would diff a model against - the wrong table. + Order matters — full identity, then last two segments, then bare name. The + live map drops ambiguous short keys, so a miss on a short candidate means + "could be either table"; returning None is correct there, where an arbitrary + match would diff against the wrong table. """ candidates = [sql_table] parts = sql_table.split(".") @@ -2175,12 +2147,10 @@ async def _collect_sql_table_diffs( """ if not sql_table_models: return {} - # The schema set is derived from the models being validated, plus the - # datasource's configured default. Introspecting only the default schema - # made every non-default-schema model unresolvable, and an unresolvable - # model is a ``WholeModelDelete`` that ``validate-models --force-clean`` - # acts on — so getting this set wrong is a data-loss path, not a - # false-positive nuisance. + # Schema set = the models' own schemas plus the datasource default. + # Introspecting only the default made non-default-schema models + # unresolvable, i.e. a ``WholeModelDelete`` — a data-loss path, so this set + # must be right. schemas: set[str | None] = { split_sql_table(m.sql_table)[0] for m in sql_table_models if m.sql_table } diff --git a/slayer/mcp/server.py b/slayer/mcp/server.py index 90e570c3..174fa3ba 100644 --- a/slayer/mcp/server.py +++ b/slayer/mcp/server.py @@ -95,11 +95,9 @@ def _fetch_tables( Returns ``(objects, None)`` on success or ``(None, friendly_error_message)`` on failure. ``schema_name=None`` uses the dialect's default schema. - Views are always included here, independent of the ingest-side - ``--no-views`` flag. This helper backs ``describe_datasource`` and the - empty-ingest probe, and a views-only schema previously reported "No tables - found — try another schema", misdirecting the agent away from a schema - that was in fact full of objects. + Views are always included (independent of ``--no-views``): this backs + ``describe_datasource`` and the empty-ingest probe, and a views-only schema + previously misreported as "No tables found". """ try: from slayer.sql import engine_factory @@ -116,10 +114,10 @@ def _fetch_tables( def _csv_arg(value: str) -> list[str] | None: - """Split a comma-separated tool argument, or None when it is empty. + """Split a comma-separated tool argument, or None when empty. - MCP tool arguments are flat scalars, so lists travel as CSV — matching - ``include_tables``' existing style rather than adding a second convention. + MCP tool args are flat scalars, so lists travel as CSV (matching + ``include_tables``). """ items = [part.strip() for part in (value or "").split(",")] return [item for item in items if item] or None @@ -1383,9 +1381,8 @@ async def create_datasource( if not auto_ingest: return "\n".join(lines) - # Auto-ingest models. The scope is passed explicitly rather than left - # to the persisted ``schema_name``, so the two can never be read as a - # conflict. + # Scope is passed explicitly, not left to the persisted ``schema_name``, + # so the two can't be read as a conflict. try: models = _ingest( datasource=ds, diff --git a/slayer/storage/type_refinement.py b/slayer/storage/type_refinement.py index aa15f0d3..f70d925b 100644 --- a/slayer/storage/type_refinement.py +++ b/slayer/storage/type_refinement.py @@ -140,9 +140,8 @@ def _parse_sql_table_with_default_schema( ``datasource.schema_name`` when the name is unqualified. This honours attached SQLite schemas instead of silently using ``main``. - The schema is everything before the FINAL dot, so a hand-written - Snowflake ``db.schema.table`` or BigQuery ``project.dataset.table`` keeps - its catalog rather than losing it to a split on the first dot. + Splits on the FINAL dot, so a Snowflake ``db.schema.table`` or BigQuery + ``project.dataset.table`` keeps its catalog. """ default_schema = getattr(datasource, "schema_name", None) or None schema_name, table_name = split_sql_table(sql_table) diff --git a/slayer/storage/v8_migration.py b/slayer/storage/v8_migration.py index ab6cf235..5536c258 100644 --- a/slayer/storage/v8_migration.py +++ b/slayer/storage/v8_migration.py @@ -1,16 +1,12 @@ """v7 → v8 schema migration for SlayerModel. -v8 introduces one new optional field on ``SlayerModel``: +v8 adds one optional field, ``source_kind`` +(``Literal["table", "view", "materialized_view"] | None``) — what kind of +object ``sql_table`` names. Auto-ingestion sets it; other models leave it None. -- ``source_kind: Optional[Literal["table", "view", "materialized_view"]]`` — - what kind of database object ``sql_table`` names. Auto-ingestion sets it; - hand-authored, ``sql``-mode and query-backed models leave it ``None``. - -The forward conversion is a no-op because the field defaults to ``None`` on -the Pydantic class. ``None`` is also the *correct* value for a pre-v8 model: -we genuinely do not know what backed it, and guessing ``"table"`` would be a -fabrication that the next re-ingest would silently have to correct. The first -subsequent ingest classifies it for real. +Forward conversion is a no-op: the field defaults to ``None``, which is also +correct for a pre-v8 model (its backing object is genuinely unknown, and the +next ingest classifies it for real). """ from __future__ import annotations diff --git a/tests/test_docs_duckdb_install.py b/tests/test_docs_duckdb_install.py index b79636ff..c19a2f6e 100644 --- a/tests/test_docs_duckdb_install.py +++ b/tests/test_docs_duckdb_install.py @@ -1,10 +1,8 @@ """Docs must not advertise a `motley-slayer[duckdb]` extra. -It does not exist: `duckdb`/`duckdb-engine` are unconditional core deps, so -the extra is warned about and ignored. Deliberately NOT fixed in pyproject — -listing them under `[tool.poetry.extras]` would *gate* rather than alias them, -breaking bare `pip install motley-slayer`, and duckdb is load-bearing for the -Postgres facade. +It does not exist: `duckdb`/`duckdb-engine` are unconditional core deps. +Deliberately NOT "fixed" in pyproject — listing them under extras would *gate* +rather than alias them, breaking bare `pip install motley-slayer`. """ from __future__ import annotations @@ -34,9 +32,8 @@ def test_no_doc_advertises_a_duckdb_extra(path: Path) -> None: def test_duckdb_remains_a_core_dependency() -> None: - """Pins the premise of the docs fix. If duckdb ever legitimately becomes - an extra, this fails and the docs must change back — deliberately coupled - so the two cannot drift apart.""" + """Pins the docs-fix premise: if duckdb ever becomes an extra this fails, + so the docs must change back with it.""" data = tomllib.loads((_REPO_ROOT / "pyproject.toml").read_text()) deps = data["tool"]["poetry"]["dependencies"] diff --git a/tests/test_ingest_cli_ux.py b/tests/test_ingest_cli_ux.py index 9ad20922..7a3b12db 100644 --- a/tests/test_ingest_cli_ux.py +++ b/tests/test_ingest_cli_ux.py @@ -1,12 +1,9 @@ """`slayer ingest` must not be silent. -Ingest printed only additions, drift, and errors — zero of each meant no -output and exit 0, so a views-only schema looked like success. - -Exit-code contract: skipped objects or an empty scan both exit 1. But -`POST /ingest` keeps 422 for `errors` only — a permanent 422 aimed at a -machine that cannot act on the `--exclude` hint would bury a successful -partial ingest behind an error status. +Zero additions/drift/errors used to mean no output and exit 0, so a +views-only schema looked like success. Exit contract: skipped objects or +an empty scan both exit 1. But `POST /ingest` keeps 422 for `errors` only, +so a partial ingest with skips is not buried behind an error status. """ from __future__ import annotations @@ -215,8 +212,7 @@ def test_skipped_section_lists_every_entry_with_its_reason( assert "ambiguous" in out def test_skipped_and_errors_are_reported_separately(self, capsys) -> None: - """A skip ('cannot model this object') and an error ('failed to save - this model') have different causes and different fixes.""" + """Skip and error have different causes and fixes, so report them separately.""" from slayer.engine.schema_drift import IngestionError result = IdempotentIngestResult( @@ -238,8 +234,7 @@ def test_skipped_and_errors_are_reported_separately(self, capsys) -> None: assert skipped_at != errors_at def test_empty_message_wording_covers_views(self) -> None: - """The old wording said 'No tables found', which is exactly what - misled the reporter into thinking the schema was empty.""" + """Old wording 'No tables found' misled the reporter into thinking the schema was empty.""" ds = SimpleNamespace(name="ds", type="sqlite", database=":memory:") msg = _empty_ingest_message(schema_name="analytics", ds=ds) assert "views" in msg.lower() @@ -255,8 +250,7 @@ class TestRestExitSemantics: def test_skipped_only_returns_200_with_skipped_in_body( self, monkeypatch, workspace: Path ) -> None: - """D12. Skips must NOT turn into 422; the body carries them - so a client can see what happened alongside the successful additions.""" + """D12. Skips must NOT turn into 422; the body carries them alongside additions.""" from fastapi.testclient import TestClient from slayer.api.server import create_app @@ -338,8 +332,7 @@ async def _fake(**_kwargs): class TestDatasourcesCreateCarveOut: def test_empty_db_still_exits_zero(self, workspace: Path, capsys) -> None: - """D14. Creating the datasource is this command's job and it - succeeded; an empty database must not fail it.""" + """D14. Creating the datasource succeeded, so an empty database must not fail it.""" import sqlite3 from slayer.cli import _run_datasources_create @@ -366,8 +359,7 @@ def test_empty_db_still_exits_zero(self, workspace: Path, capsys) -> None: assert "No models were generated." in capsys.readouterr().out class TestViewsFlagWiring: - """The parser is built inline in ``main()``, so drive it through ``main()`` - with a stubbed handler that captures the parsed args.""" + """Parser is built inline in ``main()``, so drive it through ``main()`` with a stub that captures args.""" @staticmethod def _capture(monkeypatch, argv: list[str], handler: str) -> SimpleNamespace: @@ -401,8 +393,7 @@ def test_ingest_accepts_no_views_flag(self, monkeypatch) -> None: assert args.include_views is False def test_datasources_create_defaults_to_views_on(self, monkeypatch) -> None: - """D15 — the flag exists on this subcommand too, so behaviour is - consistent with `slayer ingest`.""" + """D15 — the flag exists on this subcommand too, matching `slayer ingest`.""" args = self._capture( monkeypatch, ["datasources", "create", "sqlite:///x.db", "--ingest"], diff --git a/tests/test_ingestion.py b/tests/test_ingestion.py index 50fda67d..b463f1dd 100644 --- a/tests/test_ingestion.py +++ b/tests/test_ingestion.py @@ -52,10 +52,8 @@ class TestGetColumnsFallback: """Tests for _get_columns_fallback parameterized queries.""" def test_without_schema(self): - """With no schema the query cannot be narrowed, so it selects the - catalog and schema alongside the columns and groups the rows by them. - Unioning every match — which is what selecting only the columns did — - produced models referencing columns their table does not have.""" + """With no schema the query selects catalog+schema alongside the columns + and groups rows by them, instead of unioning matches across schemas.""" engine, conn = _setup_mock_engine( [ ("db", "public", "id", "INTEGER"), diff --git a/tests/test_ingestion_name_sanitize.py b/tests/test_ingestion_name_sanitize.py index 31b4ae22..324e9e34 100644 --- a/tests/test_ingestion_name_sanitize.py +++ b/tests/test_ingestion_name_sanitize.py @@ -1,12 +1,9 @@ """One bad table name must not abort the whole ingest. dlt flattens nested JSON into ``__``-named child tables, which model names -reserve for join-path aliases. The ``ValidationError`` was raised inside the -table loop, which runs wholly before the per-table isolation downstream — so -the run died before creating a single model. - -Fix: sanitize ``__`` into the model name (``sql_table`` keeps the real name), -plus a ``try/except`` backstop for everything else. +reserve for join-path aliases; the ``ValidationError`` fired before a single +model was created. Fix: sanitize ``__`` into the model name (``sql_table`` +keeps the real name), plus a ``try/except`` backstop for the rest. """ from __future__ import annotations @@ -61,8 +58,7 @@ class TestSanitizer: [ ("reports__patient__drug", "reports_patient_drug"), ("a__b", "a_b"), - # A naive str.replace("__", "_") is non-overlapping left-to-right - # and leaves "a__b" here — which still fails validation. + # A naive str.replace("__", "_") would leave "a__b" here. ("a___b", "a_b"), ("a____b", "a_b"), ("a_____b", "a_b"), @@ -89,17 +85,15 @@ def test_is_idempotent(self, raw: str) -> None: assert sanitize_model_name(once) == once def test_result_is_a_valid_model_name(self) -> None: - """The whole point: the output must pass the validator that rejected - the input.""" + """Output must pass the validator that rejected the input.""" name = sanitize_model_name("reports__patient__drug") model = SlayerModel(name=name, sql_table="reports__patient__drug", data_source="ds") assert model.name == "reports_patient_drug" def test_does_not_touch_other_reserved_characters(self) -> None: - """Only ``__`` is sanitized. Dots stay put — a dotted table name makes - ``sql_table`` itself ambiguous with schema qualification, so those go - down the skip path instead.""" + """Only ``__`` is sanitized; dots stay put (ambiguous with schema + qualification, so those go down the skip path instead).""" assert sanitize_model_name("weird.table") == "weird.table" assert sanitize_model_name("odd:name") == "odd:name" @@ -113,8 +107,7 @@ class TestDunderTableIngestion: def test_dunder_table_is_modelled_under_a_sanitized_name( self, workspace: Path ) -> None: - """sql_table keeps the real object name so queries still - resolve.""" + """sql_table keeps the real object name so queries still resolve.""" ds = _sqlite_ds( workspace, """ @@ -129,8 +122,7 @@ def test_dunder_table_is_modelled_under_a_sanitized_name( assert model.sql_table == "reports__patient__drug" def test_one_bad_name_no_longer_kills_the_run(self, workspace: Path) -> None: - """THE headline regression. Before the fix this raised a - ValidationError and produced zero models.""" + """THE headline regression: before the fix this produced zero models.""" ds = _sqlite_ds( workspace, """ @@ -161,8 +153,7 @@ def test_dunder_view_is_also_sanitized(self, workspace: Path) -> None: def test_schema_qualified_dunder_table_keeps_qualified_sql_table( self, workspace: Path ) -> None: - """The schema prefix must survive sanitization untouched — only the - model name changes.""" + """The schema prefix survives sanitization; only the model name changes.""" ds = _sqlite_ds( workspace, "CREATE TABLE reports__patient (id INTEGER PRIMARY KEY, x TEXT);", @@ -186,8 +177,7 @@ class TestCollisionPolicy: def test_real_table_wins_and_dunder_table_is_skipped( self, workspace: Path ) -> None: - """an unsanitized name always beats a sanitized one, so the - model named ``a_b`` is the one actually called ``a_b`` in the DB.""" + """An unsanitized name always beats a sanitized one for the same model name.""" ds = _sqlite_ds(workspace, self._SCRIPT) report = ingest_datasource_report(datasource=ds) @@ -204,8 +194,8 @@ def test_collision_skip_records_a_reason(self, workspace: Path) -> None: assert "a_b" in entry.reason def test_collision_outcome_is_order_independent(self, workspace: Path) -> None: - """determinism. Reversing the scan order must not flip which - object gets the name; otherwise re-ingest churns models.""" + """Determinism: reversing scan order must not flip the winner, else + re-ingest churns models.""" from slayer.engine import ingestion as ingestion_module ds = _sqlite_ds(workspace, self._SCRIPT) @@ -228,8 +218,7 @@ def _mapping(report): assert _skipped_names(forward) == _skipped_names(backward) def test_no_numeric_suffix_disambiguation(self, workspace: Path) -> None: - """Suffixes were rejected: they are unstable across runs as the table - set changes, orphaning models and churning drift.""" + """Numeric suffixes are unstable across runs, orphaning models.""" ds = _sqlite_ds(workspace, self._SCRIPT) report = ingest_datasource_report(datasource=ds) assert not any(m.name.endswith(("_2", "_3")) for m in report.models) @@ -242,8 +231,7 @@ def test_no_numeric_suffix_disambiguation(self, workspace: Path) -> None: def test_two_dunder_names_collapsing_to_one_skips_the_second( self, workspace: Path ) -> None: - """``a__b`` and ``a___b`` both sanitize to ``a_b`` with no real ``a_b`` - present — exactly one wins, the other is skipped.""" + """Both sanitize to ``a_b`` with no real ``a_b``; one wins, one is skipped.""" ds = _sqlite_ds(workspace, self._TWO_DUNDER) report = ingest_datasource_report(datasource=ds) assert len([m for m in report.models if m.name == "a_b"]) == 1 @@ -252,14 +240,8 @@ def test_two_dunder_names_collapsing_to_one_skips_the_second( def test_sanitized_vs_sanitized_winner_is_order_independent( self, workspace: Path ) -> None: - """Two objects collapsing to the SAME model name must pick the same - winner whatever order the inspector lists them in. - - Distinct from the real-vs-sanitized case above: here neither name is - reserved up front, so a naive first-come rule lets the scan order decide - which physical table `a_b` queries — the same instability that ruled out - numeric suffixes. - """ + """Two objects collapsing to the same model name pick the same winner + regardless of scan order — here neither name is reserved up front.""" from slayer.engine import ingestion as ingestion_module ds = _sqlite_ds(workspace, self._TWO_DUNDER) @@ -291,8 +273,7 @@ class TestSkipBackstop: def test_unmodellable_object_is_skipped_not_fatal( self, workspace: Path, monkeypatch ) -> None: - """anything that fails per-object construction is skipped - with the rest of the run intact.""" + """Anything that fails per-object construction is skipped, run intact.""" from slayer.engine import ingestion as ingestion_module ds = _sqlite_ds( @@ -321,10 +302,8 @@ def _explode(*args, **kwargs): def test_fk_introspection_failure_does_not_abort_the_run( self, workspace: Path, monkeypatch ) -> None: - """``_get_fk_relationships`` and the FK-collection loop both - run BEFORE the per-object try/except, so an unguarded raise there - would kill the run regardless of the backstop. Views legitimately have - no FKs and some dialects raise rather than returning [].""" + """FK introspection runs BEFORE the per-object try/except, so an + unguarded raise there kills the run; some dialects raise for views.""" ds = _sqlite_ds( workspace, """ @@ -366,12 +345,8 @@ class TestEngineDisposal: def test_dispose_failure_does_not_mask_the_real_error( self, workspace: Path, monkeypatch ) -> None: - """Disposal runs in a ``finally``; if it raises there it would replace - the in-flight exception, and the caller would see a teardown error - instead of the driver error that actually failed the run. The REST - layer surfaces that exception's message, so masking it is a real - diagnosability loss. - """ + """A raising dispose in the ``finally`` must not replace the in-flight + exception, else the caller sees the teardown error, not the real one.""" from slayer.engine import ingestion as ingestion_module ds = _sqlite_ds( @@ -405,8 +380,7 @@ def dispose(self): def test_dispose_failure_does_not_fail_a_successful_ingest( self, workspace: Path, monkeypatch ) -> None: - """The other half: with no in-flight exception, a raising dispose in - the ``finally`` would turn a completed ingest into a failure.""" + """The other half: a raising dispose must not fail a completed ingest.""" ds = _sqlite_ds( workspace, "CREATE TABLE orders (id INTEGER PRIMARY KEY, x TEXT);" ) @@ -434,17 +408,14 @@ def _explode(): assert {m.name for m in report.models} == {"orders"} assert disposed, "dispose must still be attempted" finally: - # The stub swallowed the real disposal, and engine_factory caches - # engines, so without this the pool would hold the SQLite file open - # for the rest of the session. + # engine_factory caches engines; without this the pool holds the file open. if real_dispose is not None: real_dispose() def test_dispose_failure_is_logged_at_warning( self, workspace: Path, caplog ) -> None: - """Disposal releases the connection that otherwise blocks external - access to the same file, so a failure must be visible above DEBUG.""" + """A dispose failure must be visible above DEBUG.""" from slayer.engine.ingestion import _dispose_quietly class _ExplodingEngine: @@ -490,8 +461,7 @@ def test_report_carries_models_skipped_and_objects( assert {o.name for o in report.objects} == {"orders", "v_orders"} def test_empty_schema_reports_no_objects(self, workspace: Path) -> None: - """Distinguishing 'schema was empty' from 'everything was skipped' is - what drives the CLI's exit-1-with-hint path.""" + """Empty schema vs everything-skipped drives the CLI's exit-1-with-hint path.""" db_path = str(workspace / "empty.db") sqlite3.connect(db_path).close() ds = DatasourceConfig(name="ds", type="sqlite", database=db_path) diff --git a/tests/test_ingestion_schema_qualification.py b/tests/test_ingestion_schema_qualification.py index c8f86a22..7081171b 100644 --- a/tests/test_ingestion_schema_qualification.py +++ b/tests/test_ingestion_schema_qualification.py @@ -1,28 +1,10 @@ """Ingested models must keep enough schema information to be queryable. -A bare ``slayer ingest`` against DuckDB swept *every* schema and wrote each -object's bare name into ``sql_table``, so a model in a non-default schema -generated ``FROM reports`` and failed with a table-not-found error. Models in -the connection's default schema resolved via the search path, which is what -made the breakage look partial rather than systemic. - -Three defects are pinned here: - -* **D1** — cross-schema discovery with no schema recorded. -* **D2** — same-named tables in two schemas silently merged their columns, - because the ``INFORMATION_SCHEMA`` column fallback ran unfiltered. -* **D3** — ``datasources create --schema X --ingest`` discarded ``schema_name`` - while ``validate-models`` read it back. - -The token discipline these tests enforce is easy to get backwards, so it is -worth stating: on DuckDB ``get_schema_names()`` returns **catalog-qualified** -tokens, and the qualified token is the *safe* one. Measured, a bare ``main`` -token makes ``get_table_names`` and ``has_table`` reach into ``ATTACH``ed -catalogs and makes the column fallback return the union across catalogs. So the -discovery token stays qualified end to end, and the column fallback filters on -``table_catalog`` as well as ``table_schema``. Separately — and this is a -different string — the qualifier written into ``sql_table`` is the bare last -segment, because the connection's current catalog is already the right one. +DuckDB ``ingest`` swept every schema and wrote bare names, so a non-default +model generated ``FROM reports`` and failed. Pinned: D1 discovery recorded no +schema; D2 same-named tables merged columns via an unfiltered fallback; D3 +``--schema X`` discarded ``schema_name``. Discovery tokens stay qualified (bare +``main`` reaches ``ATTACH``ed catalogs); ``sql_table`` gets the bare segment. """ from __future__ import annotations @@ -115,8 +97,7 @@ def _seed_collide(db_path: str) -> None: def _seed_three_schemas(db_path: str) -> None: - """``main.only_main`` + ``s2.reports`` + ``s3.reports`` — collision between - two *non-default* schemas, so the default-wins rule cannot decide it.""" + """Collision between two non-default schemas, so default-wins can't decide it.""" con = duckdb.connect(db_path) con.execute("CREATE TABLE only_main(z INTEGER)") con.execute("CREATE SCHEMA s2") @@ -148,11 +129,8 @@ def _inspector_for(ds: DatasourceConfig) -> tuple[sa.Engine, sa.engine.Inspector # --- attached-catalog fixture ---------------------------------------------- -# -# The primary file is ``att_main.duckdb`` (so the current catalog is -# ``att_main``) and the attached one is registered as ``aaa`` — deliberately -# sorting BEFORE the default catalog, so a "lowest sorted wins" tie-break would -# pick the wrong one and be caught. +# ``aaa`` is attached sorting BEFORE the current catalog ``att_main`` so a +# lowest-sorted tie-break would pick wrong and get caught. def _seed_attached_pair(tmp_path: Path) -> tuple[str, str]: @@ -174,11 +152,8 @@ def _seed_attached_pair(tmp_path: Path) -> tuple[str, str]: def _attached_engine(main_path: str, other_path: str) -> sa.Engine: - """A DuckDB engine on ``main_path`` with ``other_path`` attached as ``aaa``. - - ``StaticPool`` so the single DBAPI connection carrying the ``ATTACH`` - is the one every later ``Inspector`` call reuses. - """ + """DuckDB engine on ``main_path`` with ``other_path`` attached as ``aaa``; + ``StaticPool`` keeps the one connection carrying the ``ATTACH``.""" eng = sa.create_engine(f"duckdb:///{main_path}", poolclass=StaticPool) with eng.connect() as conn: conn.exec_driver_sql(f"ATTACH '{other_path}' AS aaa") @@ -242,10 +217,8 @@ async def _count(engine: SlayerQueryEngine, model_name: str) -> int: class TestReportedRegression: async def test_bare_ingest_covers_only_the_default_schema(self, tmp_path): - """Test 1. Bare ingest must resolve to the connection's default schema - only — DuckDB is the one Tier-1 dialect whose ``schema=None`` sweeps - every schema, which is how ``openfda_rest.reports`` got a bare - ``sql_table``.""" + """Test 1. Bare ingest resolves to the default schema only; DuckDB's + ``schema=None`` otherwise sweeps every schema.""" ds = _repro_ds(tmp_path) models = ingest_datasource(datasource=ds) @@ -253,8 +226,8 @@ async def test_bare_ingest_covers_only_the_default_schema(self, tmp_path): assert _by_name(models)["in_default"].sql_table == "in_default" async def test_bare_ingest_reports_the_other_schemas(self, tmp_path): - """Test 1 (cont). Narrowing the scan must not silently lose models — - the user is told which schemas were left out and how to get them.""" + """Test 1 (cont). Narrowing the scan reports left-out schemas rather + than silently losing them.""" ds = _repro_ds(tmp_path) eng, insp = _inspector_for(ds) try: @@ -273,9 +246,8 @@ async def test_bare_ingest_reports_the_other_schemas(self, tmp_path): assert scope.other_schemas == ["openfda_rest"] async def test_explicit_schema_qualifies_and_is_queryable(self, tmp_path): - """Test 2. The issue's exact repro: ingest the non-default schema and - run ``*:count`` against it. Before the fix the generated SQL is - ``FROM reports`` and DuckDB raises a catalog error.""" + """Test 2. Exact repro: ingesting a non-default schema then ``*:count`` + emitted ``FROM reports`` and raised a catalog error before the fix.""" ds = _repro_ds(tmp_path) storage = _storage(tmp_path) result = await _ingest(ds, storage, schemas=["openfda_rest"]) @@ -286,9 +258,8 @@ async def test_explicit_schema_qualifies_and_is_queryable(self, tmp_path): assert await _count(SlayerQueryEngine(storage=storage), "reports") == 2 async def test_all_schemas_qualifies_only_non_default(self, tmp_path): - """Test 3. Default-schema objects stay unqualified so turning the flag - on never rewrites models that already exist; everything else is - qualified. Both must be queryable.""" + """Test 3. Default-schema objects stay unqualified (so the flag never + rewrites existing models); everything else is qualified and queryable.""" ds = _repro_ds(tmp_path) storage = _storage(tmp_path) result = await _ingest(ds, storage, all_schemas=True) @@ -313,10 +284,8 @@ class TestColumnCorruption: async def test_bare_ingest_does_not_union_columns_across_schemas( self, tmp_path ): - """Test 4. ``main.reports(a)`` and ``s2.reports(b, c)`` are different - tables. The schema-blind ``INFORMATION_SCHEMA`` query returned all - three columns, producing a model that references columns its table - does not have.""" + """Test 4. A schema-blind ``INFORMATION_SCHEMA`` query unioned the + columns of ``main.reports(a)`` and ``s2.reports(b, c)`` into one model.""" ds = _collide_ds(tmp_path) models = ingest_datasource(datasource=ds) @@ -360,9 +329,8 @@ class TestCollisions: async def test_all_schemas_collision_gives_the_default_schema_the_name( self, tmp_path ): - """Test 7. Two schemas claim model name ``reports``. The default - schema wins; the loser is skipped, never suffixed — suffixes shift - with the object set and orphan models.""" + """Test 7. When two schemas claim ``reports`` the default wins and the + loser is skipped, never suffixed (suffixes shift and orphan models).""" ds = _collide_ds(tmp_path) report = ingest_datasource_report(datasource=ds, all_schemas=True) @@ -391,10 +359,8 @@ async def test_non_default_collision_is_order_independent(self, tmp_path): } def test_single_schema_sanitization_behaviour_is_unchanged(self, tmp_path): - """Test 9. With one schema in scope the cross-schema tie-break keys - never fire, so ``__`` sanitization stays byte-identical. The real - guard is tests/test_ingestion_name_sanitize.py passing unchanged; - this pins the same rule at the helper.""" + """Test 9. With one schema in scope the cross-schema tie-break never + fires, so ``__`` sanitization stays byte-identical.""" objects = [ IngestableObject(name="a_b", kind="table", schema="main"), IngestableObject(name="a__b", kind="table", schema="main"), @@ -436,9 +402,8 @@ class TestSelfHeal: async def test_missing_qualifier_is_healed_and_metadata_preserved( self, tmp_path ): - """Test 10. Re-ingest repairs a *missing* qualifier. It usually changes - no columns and no joins, so the repair has to participate in the - short-circuit or the merged model is computed and discarded.""" + """Test 10. Re-ingest repairs a missing qualifier and preserves + metadata; the repair must join the no-change short-circuit.""" ds = _repro_ds(tmp_path) storage = _storage(tmp_path) await storage.save_datasource(ds) @@ -508,9 +473,8 @@ async def test_default_schema_model_is_untouched(self, tmp_path): class TestCrossSchemaMergeGuard: async def test_sequential_single_schema_ingests_do_not_fuse(self, tmp_path): - """Test 13. Two explicit single-schema ingests must not merge two - different physical tables into one model. No new flag is involved — - this fuses today.""" + """Test 13. Two explicit single-schema ingests must not fuse two + different physical tables into one model.""" ds = _collide_ds(tmp_path) storage = _storage(tmp_path) await _ingest(ds, storage, schemas=["main"]) @@ -524,10 +488,9 @@ async def test_sequential_single_schema_ingests_do_not_fuse(self, tmp_path): ) async def test_bare_persisted_model_is_not_repointed(self, tmp_path): - """Test 27. The hole that qualifying only non-default schemas opens: a - default-schema model is persisted unqualified, so a schema comparison - alone cannot fire. Without the live ``has_table`` probe the self-heal - happily repoints ``reports`` at ``s2.reports``.""" + """Test 27. A default-schema model is persisted unqualified, so without + the live ``has_table`` probe self-heal repoints ``reports`` at + ``s2.reports``.""" ds = _collide_ds(tmp_path) storage = _storage(tmp_path) await _ingest(ds, storage) # bare -> sql_table: reports, columns [a] @@ -554,9 +517,8 @@ class TestValidateModels: async def test_multi_schema_models_are_not_marked_for_deletion( self, tmp_path ): - """Test 14. The data-loss guard. A qualified model that the live map - cannot resolve becomes a ``WholeModelDelete``, which - ``validate-models --force-clean`` acts on.""" + """Test 14. Data-loss guard: a qualified model the live map can't + resolve becomes a ``WholeModelDelete`` that ``--force-clean`` deletes.""" ds = _repro_ds(tmp_path) storage = _storage(tmp_path) await _ingest(ds, storage, all_schemas=True) @@ -587,9 +549,8 @@ async def test_qualified_model_diffs_against_its_own_table(self, tmp_path): assert to_delete == [], to_delete async def test_view_in_a_non_default_schema_still_resolves(self, tmp_path): - """Test 16. Views are included in the live map unconditionally; the - schema-awareness change must not quietly re-arm the view blindness - that made view-backed models look deleted.""" + """Test 16. Views stay in the live map, so the schema-awareness change + must not re-arm the view blindness that made them look deleted.""" db_path = str(tmp_path / "views.duckdb") con = duckdb.connect(db_path) con.execute("CREATE SCHEMA analytics") @@ -639,15 +600,13 @@ def test_bare_table_name_takes_the_last_segment(self, sql_table, expected): ], ) def test_split_sql_table_preserves_the_catalog(self, sql_table, expected): - """Tests 18/33. The schema token is everything before the final dot. - Truncating to the last two segments would discard catalog identity — - which on DuckDB matches nothing at all.""" + """Tests 18/33. The schema token is everything before the final dot; + truncating to two segments would drop catalog identity.""" assert split_sql_table(sql_table) == expected def test_parse_with_default_schema_preserves_the_catalog(self): - """Test 33. Snowflake ``db.schema.table`` and BigQuery - ``project.dataset.table`` are hand-writable today, and ``partition`` - split them after the FIRST dot.""" + """Test 33. Snowflake/BigQuery three-part names are hand-writable, and + ``partition`` split them after the FIRST dot.""" ds = _duckdb_ds(":memory:", schema_name="fallback") assert _parse_sql_table_with_default_schema( "proj.dataset.tbl", ds @@ -661,9 +620,8 @@ def test_schema_of_returns_the_bare_segment(self): assert _schema_of("c.s.t") == "s" def test_resolve_live_table_walks_progressively_shorter_keys(self): - """Test 19. A three-part ``sql_table`` must resolve against a live map - keyed either way — and must return None rather than a wrong match when - the short key is ambiguous.""" + """Test 19. A three-part ``sql_table`` resolves against a live map keyed + either way, and returns None on an ambiguous short key.""" live = LiveTable(columns={"a": DataType.INT}) assert _resolve_live_table( sql_table="c.s.t", live_tables={"s.t": live} @@ -712,10 +670,8 @@ class TestSchemaNamePersistence: async def test_create_with_schema_persists_it_and_bare_ingest_reuses_it( self, tmp_path ): - """Test 20. ``--schema`` was used for the one-shot ingest and thrown - away, while ``validate-models`` read ``schema_name`` back — so the two - commands looked at different schemas. Drives the real CLI entry - points, which is also what pins the persist-then-ingest call order.""" + """Test 20. ``--schema`` fed the one-shot ingest but wasn't persisted + for ``validate-models`` to read back, so the two saw different schemas.""" db_path = str(tmp_path / "fda.duckdb") _seed_repro(db_path) storage = _storage(tmp_path) @@ -745,9 +701,8 @@ async def test_create_with_schema_persists_it_and_bare_ingest_reuses_it( async def test_multi_schema_create_does_not_persist_schema_name( self, tmp_path ): - """Test 21. ``schema_name`` is a single-schema default. A CSV list or - ``--all-schemas`` has no single value to persist, and persisting the - first would silently narrow every later bare ingest.""" + """Test 21. ``schema_name`` is a single-schema default; a CSV list or + ``--all-schemas`` has no single value to persist.""" db_path = str(tmp_path / "fda.duckdb") _seed_repro(db_path) @@ -964,9 +919,8 @@ async def test_conflicting_scope_arguments_are_reported( class TestUnqualifiedNonRegression: def test_sqlite_ingest_stays_unqualified(self, tmp_path): - """Test 26. SQLite reports ``main`` as its default schema. Qualifying - it would rewrite every existing model on disk for no benefit, and the - SQLite fixtures across the suite pin the unqualified form.""" + """Test 26. SQLite's default schema is ``main``; qualifying it would + rewrite every existing model on disk for no benefit.""" db_path = str(tmp_path / "live.db") conn = sqlite3.connect(db_path) conn.executescript( @@ -996,9 +950,8 @@ def test_duckdb_default_schema_stays_unqualified(self, tmp_path): class TestAttachedCatalogs: def test_all_schemas_covers_only_the_current_catalog(self, attached): - """Test 28. ``--all-schemas`` means "this database", not "and whatever - anyone attached to the session". The dropped schemas are reported, not - silently discarded.""" + """Test 28. ``--all-schemas`` means this database, not attached + catalogs; dropped schemas are reported, not silently discarded.""" report = ingest_datasource_report( datasource=attached.ds, all_schemas=True ) @@ -1011,8 +964,7 @@ def test_all_schemas_covers_only_the_current_catalog(self, attached): dropped = [s for s in report.skipped if "attached catalog" in s.reason] assert dropped, report.skipped - # The reason has to be actionable: it must name the catalog AND the - # exact invocation that would ingest it. + # The reason must name the catalog AND the invocation that ingests it. reason = next(s.reason for s in dropped if "aaa" in s.reason) assert "aaa.main" in reason assert "--schema" in reason @@ -1020,9 +972,8 @@ def test_all_schemas_covers_only_the_current_catalog(self, attached): def test_bare_ingest_does_not_reach_into_an_attached_catalog( self, attached ): - """Test 29. ``get_table_names(schema="main")`` still returns - ``only_in_other`` from the attached catalog — narrowing to the bare - default schema is not enough, the token must carry the catalog.""" + """Test 29. ``get_table_names(schema="main")`` still returns attached + ``only_in_other``, so the token must carry the catalog.""" report = ingest_datasource_report(datasource=attached.ds) names = _by_name(report.models) @@ -1031,9 +982,8 @@ def test_bare_ingest_does_not_reach_into_an_attached_catalog( assert [c.name for c in names["shared"].columns] == ["m"] def test_column_fallback_accepts_a_qualified_token(self, attached): - """Test 30. The measured zero-column trap: filtering on - ``table_schema`` alone means a qualified token matches nothing and the - model is created silently empty.""" + """Test 30. Filtering on ``table_schema`` alone makes a qualified token + match nothing, creating a silently empty model.""" eng = attached.engine() try: cols = _get_columns_fallback(eng, "reports", "att_main.openfda_rest") @@ -1042,10 +992,8 @@ def test_column_fallback_accepts_a_qualified_token(self, attached): assert [c["name"] for c in cols] == ["id", "n"] def test_column_fallback_never_unions_across_catalogs(self, attached): - """Test 30a. ``shared`` exists in both catalogs under schema ``main``. - Normalising the token to its bare last segment — which an earlier draft - of this work proposed — returns ``['m', 'o']``. This is the test that - keeps that rule from coming back.""" + """Test 30a. ``shared`` exists in both catalogs under ``main``, so + normalising the token to its bare last segment unions to ``['m', 'o']``.""" eng = attached.engine() try: qualified = _get_columns_fallback(eng, "shared", "att_main.main") @@ -1054,18 +1002,14 @@ def test_column_fallback_never_unions_across_catalogs(self, attached): eng.dispose() assert [c["name"] for c in qualified] == ["m"] - # Pins the hazard itself, so the reason for the qualified token is - # visible in the test rather than only in the commit message. Sorted: - # the union spans two catalogs and ``ORDER BY ordinal_position`` says - # nothing about which catalog's rows come first. + # Sorted: the cross-catalog union has no deterministic column order. assert sorted(c["name"] for c in bare) == ["m", "o"] def test_column_fallback_prefers_the_default_over_the_lowest_sorted( self, attached ): - """Test 32. The attached catalog is named ``aaa`` so it sorts first. - A lowest-sorted tie-break would swap union corruption for deterministic - wrong-table corruption, which is harder to notice.""" + """Test 32. ``aaa`` sorts first, so a lowest-sorted tie-break would pick + the wrong table; the default must win instead.""" eng = attached.engine() try: cols = _get_columns_fallback( @@ -1077,8 +1021,7 @@ def test_column_fallback_prefers_the_default_over_the_lowest_sorted( def test_column_fallback_raises_when_it_cannot_disambiguate(self, attached): """Test 32 (cont). With no default to fall back on, refuse rather than - pick. Per-object isolation turns this into a reported skip, so one - ambiguous object never aborts the run.""" + pick a table; isolation turns the raise into a reported skip.""" eng = attached.engine() try: with pytest.raises(ValueError) as excinfo: @@ -1092,11 +1035,8 @@ def test_column_fallback_raises_when_it_cannot_disambiguate(self, attached): def test_a_bare_requested_schema_is_upgraded_before_discovery( self, attached ): - """A user types ``--schema main``, and a bare token reaches into the - ATTACHed catalog exactly as ``schema=None`` used to. The token is - upgraded to the enumerated ``att_main.main`` for discovery — but the - emitted qualifier stays the ``main`` the user asked for, because - resolving for introspection must not change the SQL we persist.""" + """``--schema main`` upgrades to ``att_main.main`` for discovery (a bare + token reaches the ATTACHed catalog) but still emits the bare ``main``.""" report = ingest_datasource_report(datasource=attached.ds, schemas=["main"]) names = _by_name(report.models) @@ -1104,9 +1044,8 @@ def test_a_bare_requested_schema_is_upgraded_before_discovery( assert names["in_default"].sql_table == "main.in_default" def test_a_bare_schema_is_upgraded_for_validation_too(self, attached): - """``validate-models`` derives its schema set from persisted - ``sql_table`` values, which carry the BARE schema — so it hits the - same hazard from the other direction.""" + """``validate-models`` derives its schema set from persisted BARE + ``sql_table`` values, hitting the same hazard from the other direction.""" live = _live_schema_for_datasource( datasource=attached.ds, schemas=["openfda_rest"] ) @@ -1116,10 +1055,8 @@ def test_a_bare_schema_is_upgraded_for_validation_too(self, attached): assert not any(k.startswith("aaa.") for k in live), sorted(live) def test_primary_key_is_not_duplicated_across_catalogs(self, tmp_path): - """DuckDB names a PK constraint after its column, so a same-shaped - table in an ATTACHed catalog gets the SAME auto-generated name. The - INFORMATION_SCHEMA join has to carry the catalog or it matches both - and returns the column twice.""" + """DuckDB names a PK constraint after its column, so the join must carry + the catalog or a same-shaped ATTACHed table returns the column twice.""" main_path = str(tmp_path / "att_main.duckdb") other_path = str(tmp_path / "att_other.duckdb") for path, extra in ((main_path, "a"), (other_path, "b")): @@ -1169,21 +1106,14 @@ def test_an_unknown_name_is_left_alone(self, tmp_path): eng, insp = _inspector_for(ds) try: assert resolve_schema_token(insp, "nope") == "nope" - # A catalog-qualified name we do not recognise stays verbatim - # rather than being second-guessed. + # An unrecognised catalog-qualified name stays verbatim. assert resolve_schema_token(insp, "other.main") == "other.main" finally: eng.dispose() def test_a_dot_in_a_bare_dialects_schema_name_is_not_a_catalog(self): - """Postgres allows ``CREATE SCHEMA "foo.bar"`` and lists schema names - BARE, so a dot there belongs to the name. Reading it as a catalog - separator would silently resolve a request for the nonexistent ``bar`` - into ``foo.bar`` and ingest a schema the user never asked for. - - Mocked because no dialect SLayer tests against can hold both - properties at once — which is exactly why it needs pinning. - """ + """Postgres lists schema names BARE, so a dot in ``"foo.bar"`` is part of + the name, not a catalog separator (mocked; no tested dialect fits).""" insp = MagicMock(spec=sa.engine.Inspector) insp.get_schema_names.return_value = ["public", "foo.bar"] insp.default_schema_name = "public" @@ -1209,12 +1139,8 @@ def test_sqlite_tokens_pass_through_unchanged(self, tmp_path): def test_a_qualifying_dialect_is_recognised_without_a_current_catalog( self, ): - """The gate must not be decided by ``qualified_default_schema``, which - falls back to the BARE default when the current catalog cannot be - determined. On DuckDB with attached catalogs that fallback would - misreport the dialect as bare and re-arm the cross-catalog sweep — so - the gate asks where the dialect's own default turns up in its own - enumeration instead.""" + """The gate reads the dialect's default from its own enumeration, not + ``qualified_default_schema`` (whose bare fallback re-arms the sweep).""" insp = MagicMock(spec=sa.engine.Inspector) insp.default_schema_name = "main" insp.get_schema_names.return_value = ["aaa.main", "att_main.main", "att_main.ofr"] @@ -1224,9 +1150,8 @@ def test_a_qualifying_dialect_is_recognised_without_a_current_catalog( def test_the_current_catalog_wins_when_several_catalogs_match( self, attached ): - """``aaa`` and ``att_main`` both expose ``main``. A bare ``main`` - means the database we are connected to — the same thing the engine - does for an unqualified reference.""" + """``aaa`` and ``att_main`` both expose ``main``; a bare ``main`` means + the connected database, as an unqualified reference would.""" eng = attached.engine() try: assert resolve_schema_token(sa.inspect(eng), "main") == "att_main.main" @@ -1239,10 +1164,8 @@ class TestCollisionWithSanitization: def test_exact_name_beats_sanitized_regardless_of_schema_order( self, reverse ): - """Test 35. ``s1.a__b`` sanitizes to ``a_b`` and collides with a real - ``s2.a_b``. Resolving in one phase over final model names keeps the - "no sanitization beats sanitization" rule ahead of the schema - tie-break, and keeps the outcome independent of listing order.""" + """Test 35. ``s1.a__b`` sanitizes to ``a_b`` and collides with real + ``s2.a_b``; exact-name-beats-sanitized wins regardless of listing order.""" objects = [ IngestableObject(name="a__b", kind="table", schema="s1"), IngestableObject(name="a_b", kind="table", schema="s2"), @@ -1261,18 +1184,8 @@ class TestLiveSchemaKeying: def test_contested_short_keys_resolve_the_way_the_database_would( self, attached ): - """Test 36. Keying the live map on ``schema.table`` alone lets one - catalog's entry overwrite another's, so full keys are always present - and shorter aliases are earned. - - A contested alias goes to the DEFAULT schema's entry, because that is - exactly what the database does: ``FROM shared`` and ``FROM - main.shared`` both land in the current catalog. Dropping the alias - instead looked safer and was not — every default-schema model is - persisted UNQUALIFIED, so the moment another catalog held a same-named - table the legacy model stopped resolving, and an unresolvable model is - a ``WholeModelDelete`` that ``--force-clean`` deletes. - """ + """Test 36. Full keys always present; a contested short alias goes to the + DEFAULT schema, as an unqualified reference would resolve.""" live = _live_schema_for_datasource( datasource=attached.ds, schemas=["att_main.main", "aaa.main"], @@ -1292,16 +1205,8 @@ def test_contested_short_keys_resolve_the_way_the_database_would( async def test_two_requests_naming_the_same_schema_do_not_cancel_out( self, tmp_path ): - """``validate-models`` derives its schema set from persisted - ``sql_table`` values, so a datasource holding both ``orders`` (bare - ingest) and ``main.customers`` (``--schema main``) asks for ``None`` - AND ``main`` — which resolve to the same discovery token. - - Scanned twice, every object appeared as two rival claimants for its - own alias, so the tie-break saw no unique winner and dropped the - aliases of objects that have no rival at all. Both models then became - ``WholeModelDelete``s. - """ + """``None`` and ``main`` resolve to one discovery token; scanning it + twice made every object its own rival and dropped its alias.""" db_path = str(tmp_path / "dupe.duckdb") con = duckdb.connect(db_path) con.execute("CREATE TABLE orders(id INTEGER, amt INTEGER)") @@ -1315,8 +1220,7 @@ async def test_two_requests_naming_the_same_schema_do_not_cancel_out( sql_table=short, live_tables=live ) is not None, f"{short} lost its alias to a duplicate scan" - # `None` normalises to the same discovery token as `main`, so the - # schema is introspected once, not twice. + # `None` normalises to the same token as `main`: introspected once. with patch( "slayer.engine.ingestion.list_ingestable_objects", side_effect=ingestion_module.list_ingestable_objects, @@ -1343,11 +1247,8 @@ async def test_two_requests_naming_the_same_schema_do_not_cancel_out( async def test_a_legacy_unqualified_model_survives_a_same_named_table( self, tmp_path ): - """The data-loss path the alias rule exists to prevent, end to end: a - model persisted before schemas were recorded (``sql_table: orders``) - must keep resolving once another schema gains its own ``orders``. - Without the default-schema tie-break it becomes a ``WholeModelDelete`` - and ``validate-models --force-clean`` deletes it.""" + """A legacy unqualified ``orders`` must keep resolving once another + schema gains its own ``orders``, else ``--force-clean`` deletes it.""" db_path = str(tmp_path / "legacy.duckdb") con = duckdb.connect(db_path) con.execute("CREATE TABLE orders(id INTEGER, amt INTEGER)") @@ -1394,8 +1295,7 @@ def test_unambiguous_aliases_are_still_inserted(self, attached): class TestHint: def test_hint_fires_for_a_persisted_schema_name(self, tmp_path): """Test 37. Hint eligibility is "one schema in scope and others exist", - independent of whether that schema was named explicitly. A user who set - ``schema_name`` months ago still needs to hear that a schema appeared.""" + whether or not that schema was named explicitly.""" db_path = str(tmp_path / "fda.duckdb") _seed_repro(db_path) ds = _duckdb_ds(db_path, schema_name="openfda_rest") @@ -1438,9 +1338,8 @@ async def test_cli_prints_the_hint_and_still_exits_zero( class TestScopeResolution: def test_explicit_schema_is_qualified_verbatim(self): - """An explicitly-named schema is written exactly as given, so - ``--schema public`` keeps producing ``public.orders`` as it does - today — even though ``public`` is Postgres' default.""" + """An explicitly-named schema is written verbatim, so ``--schema public`` + still produces ``public.orders`` even though ``public`` is the default.""" obj = IngestableObject(name="orders", kind="table", schema="public") resolved = ResolvedSchema(name="public", explicit=True, is_default=True) assert qualify_sql_table(obj=obj, resolved=resolved) == "public.orders" @@ -1453,9 +1352,8 @@ def test_auto_default_schema_is_not_qualified(self): assert qualify_sql_table(obj=obj, resolved=resolved) == "orders" def test_auto_non_default_schema_drops_the_catalog(self): - """The emitted qualifier is the bare last segment: the connection's - current catalog is already the right one, so re-stating it would only - break if the datasource is later repointed.""" + """The emitted qualifier is the bare last segment; the connection's + current catalog is already the right one.""" obj = IngestableObject(name="reports", kind="table", schema="fda.ofr") resolved = ResolvedSchema(name="fda.ofr", explicit=False, is_default=False) assert qualify_sql_table(obj=obj, resolved=resolved) == "ofr.reports" @@ -1492,10 +1390,7 @@ def test_requested_schemas_are_marked_explicit(self, tmp_path): finally: eng.dispose() - # ``name`` is the DISCOVERY token, upgraded to the catalog-qualified - # shape the dialect enumerates — a bare token would reach into an - # ATTACHed catalog. ``requested_as`` keeps what the user typed, and is - # what gets emitted, so the upgrade cannot leak into ``sql_table``. + # ``name`` is the qualified discovery token; ``requested_as`` is emitted. assert [ (s.name.rsplit(".", 1)[-1], s.requested_as, s.explicit) for s in scope.schemas @@ -1532,8 +1427,7 @@ def test_multi_returns_objects_tagged_with_their_schema(self, tmp_path): finally: eng.dispose() - # Objects carry the resolved discovery token, so the bare schema the - # user asked for shows up qualified here. + # Objects carry the resolved discovery token, so it shows up qualified. assert {(o.name, o.schema.rsplit(".", 1)[-1]) for o in objects} == { ("in_default", "main"), ("reports", "openfda_rest"), @@ -1547,9 +1441,8 @@ def test_scope_is_a_pydantic_model(self): class TestProcessTableOutcome: def test_outcome_carries_addition_and_skip_separately(self): - """``_process_one_table`` has to be able to say "I declined this one" - as well as "here is what I did" — a skip is not an error and must not - travel as one.""" + """``_process_one_table`` reports a skip separately from an addition; a + skip is not an error and must not travel as one.""" outcome = ProcessTableOutcome( skipped=SkippedTable(table_name="s2.reports", reason="cross-schema") ) @@ -1576,10 +1469,8 @@ def _seed_fk(db_path: str) -> None: class TestForeignKeysAreSchemaAware: - """The FK graph, the join generator and the FK-column collector all take - the DISCOVERY token, never the emitted qualifier. Nothing else in the - suite exercises a foreign key, so a token mix-up there would be invisible. - """ + """FK graph, join generator and column collector all take the DISCOVERY + token; nothing else in the suite exercises a foreign key.""" def test_joins_are_generated_for_a_non_default_schema(self, tmp_path): db_path = str(tmp_path / "fk.duckdb") @@ -1609,13 +1500,8 @@ def test_fk_columns_are_excluded_from_rollup(self, tmp_path): class TestPrimaryKeysAreSchemaAware: def test_primary_key_survives_a_qualified_schema_token(self, tmp_path): - """DuckDB's Inspector reports an empty ``constrained_columns`` even - for a declared PRIMARY KEY, so the ``INFORMATION_SCHEMA`` fallback is - the path that actually runs. It filters on ``table_schema``, which - holds the BARE name — a qualified token matches nothing and drops - every primary key silently. Fan-out safety leans on - ``Column.primary_key``, so losing it is not cosmetic. - """ + """The ``INFORMATION_SCHEMA`` PK fallback filters on the BARE + ``table_schema``, so a qualified token silently drops every primary key.""" db_path = str(tmp_path / "fk.duckdb") _seed_fk(db_path) models = _by_name( @@ -1643,9 +1529,8 @@ class TestPerObjectIsolation: async def test_a_raising_column_lookup_becomes_a_skip( self, tmp_path, monkeypatch ): - """An ambiguous column lookup raises rather than guessing. That raise - must be isolated per object — one unresolvable table cannot abort the - scan — and must surface as a skip, not an error.""" + """An ambiguous column lookup raises, and that raise must surface as a + per-object skip rather than aborting the whole scan.""" ds = _repro_ds(tmp_path) real = ingestion_module._safe_get_columns @@ -1668,10 +1553,8 @@ def _raising(inspector, sa_engine, table_name, schema): class TestCrossSchemaGuardFailsClosed: - """The guard answers "is this persisted unqualified model the default - schema's table?" from a live listing. When that listing fails the answer - is UNKNOWN, and unknown has to refuse the merge — an empty list would read - as "no such default-schema object" and wave a repoint through.""" + """When the default-schema listing fails the answer is UNKNOWN, which must + refuse the merge rather than read as "no such object" and wave it through.""" @staticmethod def _models() -> tuple[SlayerModel, SlayerModel]: @@ -1695,9 +1578,8 @@ def test_unknown_default_schema_refuses_the_merge(self): assert "cross-schema" in conflict.reason def test_a_known_empty_default_schema_still_allows_the_repair(self): - """Fail-closed must not become fail-always: when the listing - succeeded and genuinely holds no such object, the qualifier repair is - the whole point of the re-ingest.""" + """Fail-closed must not become fail-always: a successful empty listing + still allows the qualifier repair.""" persisted, fresh = self._models() assert _cross_schema_conflict( model_name="reports", persisted=persisted, fresh=fresh, @@ -1706,9 +1588,8 @@ def test_a_known_empty_default_schema_still_allows_the_repair(self): class TestAdditiveMergeQualifierRules: - """Pinned directly on ``_additive_merge_existing``. Driving these through - ingestion would let the cross-schema guard skip the model before the merge - ran, so the merge rule itself would go untested.""" + """Pinned directly on ``_additive_merge_existing``; driving through ingestion + would let the cross-schema guard skip the model before the merge ran.""" @staticmethod def _model(sql_table: str, columns: list[str]) -> SlayerModel: @@ -1736,9 +1617,8 @@ def test_unqualified_persisted_table_is_healed(self): assert result.sql_table_change == "reports → openfda_rest.reports" def test_heal_alone_is_enough_to_trigger_a_save(self): - """A qualifier repair usually changes no columns and no joins, so if it - does not participate in the short-circuit the merged model is computed - and then discarded.""" + """A qualifier repair usually changes no columns or joins, so it must + join the short-circuit or the merged model is discarded.""" result = _additive_merge_existing( persisted=self._model("reports", ["id"]), fresh=self._model("openfda_rest.reports", ["id"]), @@ -1761,9 +1641,8 @@ def test_a_different_object_name_is_not_a_qualifier_repair(self): class TestAdditionRendering: def test_updated_line_names_the_qualifier_repair(self): - """The repair is the whole point of the re-ingest, so it cannot be - silent — and a repair adds no columns, so without this the line prints - nothing at all.""" + """A qualifier repair adds no columns, so without this the "Updated" + line prints nothing at all.""" buf = io.StringIO() _print_ingest_addition( ModelAddition( @@ -1822,9 +1701,8 @@ def test_user_schemas_are_kept(self, token): class TestCollisionTieBreakOrder: def test_lower_schema_wins_and_order_does_not_matter(self): - """Rule 3. Both objects are sanitization-free and neither schema is the - default, so only the schema name can decide — and it must decide the - same way whichever order the inspector listed them in.""" + """Rule 3. Neither schema is default and both are sanitization-free, so + the schema name decides, order-independently.""" forward = [ IngestableObject(name="reports", kind="table", schema="s2"), IngestableObject(name="reports", kind="table", schema="s3"), @@ -2080,8 +1958,7 @@ def test_qualified_schema_emits_the_catalog_predicate(self): assert params["schema"] == "openfda_rest" -# Re-exported for callers that patch discovery in place (the existing -# name-sanitization tests do this); asserting it here keeps that seam visible. +# Re-exported for callers that patch discovery in place; this pins that seam. def test_module_exports_discovery_helpers(): for attr in ( "list_ingestable_objects", diff --git a/tests/test_ingestion_views.py b/tests/test_ingestion_views.py index 174363f8..f86ccdb6 100644 --- a/tests/test_ingestion_views.py +++ b/tests/test_ingestion_views.py @@ -1,13 +1,10 @@ """Ingestion, drift, and MCP listing must see views. -Introspection only called ``get_table_names()``. dbt materializes staging -models as views, so a schema of them ingested nothing and printed nothing. - -Three surfaces were blind. The drift one is a data-loss bug independent of -ingest: a hand-authored model whose ``sql_table`` named a view resolved to -``live_table=None``, producing a ``WholeModelDelete`` that -``validate-models --force-clean`` acts on. Its fix is deliberately NOT gated -on ``--no-views``. +Introspection only called ``get_table_names()``, so dbt staging models +(materialized as views) ingested nothing across three surfaces. The drift +one is a data-loss bug independent of ingest: a model whose ``sql_table`` +named a view resolved to ``live_table=None`` and produced a +``WholeModelDelete``; its fix is deliberately NOT gated on ``--no-views``. """ from __future__ import annotations @@ -70,8 +67,7 @@ def _mock_inspector( views: list[str] | Exception | None = None, matviews: list[str] | Exception | None = None, ) -> MagicMock: - """An Inspector stub whose view accessors can raise, to model dialects - that do not implement them.""" + """An Inspector stub whose view accessors can raise, to model dialects that don't implement them.""" insp = MagicMock(spec=sa.engine.Inspector) insp.get_table_names.return_value = list(tables) @@ -79,9 +75,7 @@ def _maybe(value): def _call(*_args, **_kwargs): if isinstance(value, Exception): raise value - # ``or []`` matters: an omitted accessor must exercise the - # "dialect reports no objects" path, not blow up on list(None) - # and land in the generic except branch. + # ``or []``: an omitted accessor reports no objects, not blow up on list(None). return list(value or []) return _call @@ -108,8 +102,7 @@ def _kind_of(objects: list[IngestableObject], name: str) -> str | None: class TestViewIngestion: def test_view_is_ingested_by_default(self, workspace: Path) -> None: - """the headline regression. Before the fix this returned - only ``orders``.""" + """Headline regression: before the fix this returned only ``orders``.""" _, ds = _db_with_view(workspace) models = ingest_datasource(datasource=ds) by_name = {m.name: m for m in models} @@ -133,8 +126,7 @@ def test_view_model_has_columns(self, workspace: Path) -> None: assert {c.name for c in view_model.columns} == {"id", "amount", "status"} def test_view_model_has_no_joins(self, workspace: Path) -> None: - """Views carry no FK metadata, so no joins can be derived. Pinned so - a future change doesn't silently invent them.""" + """Views carry no FK metadata, so no joins can be derived.""" _, ds = _db_with_view(workspace) models = ingest_datasource(datasource=ds) view_model = next(m for m in models if m.name == "stg_orders") @@ -148,16 +140,13 @@ def test_view_model_has_no_joins(self, workspace: Path) -> None: class TestDialectTolerance: def test_get_view_names_not_implemented_is_survivable(self) -> None: - """base Inspector raises NotImplementedError on dialects that - do not implement the accessor. Tables must still list.""" + """Base Inspector raises NotImplementedError on unimplemented accessors; tables must still list.""" insp = _mock_inspector(tables=["orders"], views=NotImplementedError()) objects = list_ingestable_objects(inspector=insp, schema=None) assert _names(objects) == ["orders"] def test_get_materialized_view_names_not_implemented_is_survivable(self) -> None: - """verified against SQLAlchemy 2.0.49: the base - ``Inspector.get_materialized_view_names`` raises rather than - returning [].""" + """Base ``Inspector.get_materialized_view_names`` raises rather than returning [] (SQLAlchemy 2.0.49).""" insp = _mock_inspector( tables=["orders"], views=["v_orders"], matviews=NotImplementedError() ) @@ -165,8 +154,7 @@ def test_get_materialized_view_names_not_implemented_is_survivable(self) -> None assert _names(objects) == ["orders", "v_orders"] def test_arbitrary_accessor_failure_is_survivable(self) -> None: - """A dialect that raises something other than NotImplementedError - (driver quirk, permissions) must not abort the scan.""" + """A non-NotImplementedError failure (driver quirk, permissions) must not abort the scan.""" insp = _mock_inspector( tables=["orders"], views=RuntimeError("no view privilege") ) @@ -174,9 +162,7 @@ def test_arbitrary_accessor_failure_is_survivable(self) -> None: assert _names(objects) == ["orders"] def test_name_returned_as_both_table_and_view_appears_once(self) -> None: - """some dialects have historically returned views from - get_table_names(). First-seen wins and the object is classified as a - table.""" + """Some dialects return views from get_table_names(); first-seen wins and it is classified as a table.""" insp = _mock_inspector(tables=["orders", "v_dup"], views=["v_dup"]) objects = list_ingestable_objects(inspector=insp, schema=None) assert _names(objects) == ["orders", "v_dup"] @@ -189,8 +175,7 @@ def test_matview_also_listed_as_view_appears_once(self) -> None: assert _kind_of(objects, "mv") == "view" def test_ordering_is_tables_then_views_then_matviews(self) -> None: - """Deterministic order matters: the name-collision reservation in - item 3 depends on a stable scan order.""" + """Deterministic order matters: name-collision reservation depends on a stable scan order.""" insp = _mock_inspector( tables=["t1", "t2"], views=["v1"], matviews=["m1"] ) @@ -210,8 +195,7 @@ def test_include_views_false_skips_accessors_entirely(self) -> None: insp.get_materialized_view_names.assert_not_called() def test_schema_is_forwarded_to_every_accessor(self) -> None: - """Schema-qualified scans must not silently fall back to the default - schema for views.""" + """Schema-qualified scans must not fall back to the default schema for views.""" insp = _mock_inspector(tables=["orders"], views=["v"], matviews=["m"]) list_ingestable_objects(inspector=insp, schema="analytics") insp.get_table_names.assert_called_once_with(schema="analytics") @@ -245,8 +229,7 @@ class TestDriftSeesViews: async def test_hand_authored_view_model_is_not_marked_for_deletion( self, workspace: Path ) -> None: - """regression for the pre-existing bug. A model pointing at a - view resolved to live_table=None and produced a WholeModelDelete.""" + """Regression: a model pointing at a view resolved to live_table=None and produced a WholeModelDelete.""" _, ds = _db_with_view(workspace) model = SlayerModel( name="stg_orders", @@ -277,8 +260,7 @@ async def test_model_pointing_at_a_genuinely_missing_object_still_deletes( assert any(isinstance(e, WholeModelDelete) for e in entries) def test_drift_sees_views_unconditionally(self, workspace: Path) -> None: - """D4. The drift side takes no include_views flag; there is - no code path by which --no-views can re-arm the deletion bug.""" + """D4. Drift takes no include_views flag, so --no-views cannot re-arm the deletion bug.""" import inspect as _inspect from slayer.engine.schema_drift import _live_schema_for_datasource @@ -300,9 +282,7 @@ def test_drift_sees_views_unconditionally(self, workspace: Path) -> None: class TestMcpListing: def test_fetch_tables_includes_views(self, workspace: Path) -> None: - """describe_datasource hid views, and the empty-ingest probe - used the same helper to decide whether to tell the agent to try - another schema.""" + """describe_datasource hid views; the empty-ingest probe used the same helper.""" from slayer.mcp.server import _fetch_tables _, ds = _db_with_view(workspace) diff --git a/tests/test_migrations.py b/tests/test_migrations.py index 2575534a..9641760a 100644 --- a/tests/test_migrations.py +++ b/tests/test_migrations.py @@ -90,9 +90,8 @@ def _step2(data: dict) -> dict: def test_register_migration_rejects_duplicates(monkeypatch) -> None: - # Synthetic entity: registering against ("SlayerModel", N) breaks the - # moment N becomes a real step, as the v7→v8 bump showed. The guard under - # test is entity-agnostic, so a fake entity pins it without the coupling. + # Synthetic entity, not ("SlayerModel", N): a real step breaks when N + # becomes real (as the v7→v8 bump showed). The guard is entity-agnostic. monkeypatch.setattr(mig, "_REGISTRY", dict(mig._REGISTRY)) @mig.register_migration("_DuplicateProbeEntity", 1) diff --git a/tests/test_model_source_kind.py b/tests/test_model_source_kind.py index 59d6eba9..cf5a29c0 100644 --- a/tests/test_model_source_kind.py +++ b/tests/test_model_source_kind.py @@ -1,14 +1,11 @@ """Persist what kind of database object backs a model. -Views have no primary key, and fan-out safety leans on ``Column.primary_key``; -``source_kind`` is what explains why the PK is absent and will stay absent. - -The sharp edge is refresh. A view→table flip (dbt ``+materialized: table``) -usually changes no columns, so it must bypass all THREE guards: the early -return in ``_additive_merge_existing``, the ``model_copy(update=...)`` set, and -the save gate in ``_process_one_table``. An implementation editing only the -update dict passes every other test here and fails -``test_view_to_table_flip_refreshes``. +Views have no primary key; ``source_kind`` explains the absent PK. + +The refresh edge: a view→table flip (dbt ``+materialized: table``) changes +no columns, so it must bypass all THREE guards — the early return in +``_additive_merge_existing``, ``model_copy(update=...)``, and the save gate +in ``_process_one_table``. """ from __future__ import annotations @@ -85,8 +82,7 @@ def test_materialized_view_is_classified(self) -> None: def test_introspect_table_to_model_defaults_to_none( self, workspace: Path ) -> None: - """the dbt and OSI converters call this without a kind and - must be unaffected.""" + """dbt and OSI converters call this without a kind; must be unaffected.""" db_path, ds = _ds(workspace, _TABLE_AND_VIEW) engine = sa.create_engine(f"sqlite:///{db_path}") try: @@ -139,8 +135,7 @@ async def test_round_trips_through_sqlite_storage(self, workspace: Path) -> None assert loaded.source_kind == "materialized_view" def test_defaults_to_none(self) -> None: - """hand-authored and sql-mode models have no live object to - classify, so None (unknown) is the correct value rather than a guess.""" + """hand-authored and sql-mode models have no live object to classify — None, not a guess.""" assert ( SlayerModel(name="m", sql_table="t", data_source="ds").source_kind is None @@ -170,8 +165,7 @@ def test_current_version_is_8(self) -> None: assert SlayerModel(name="m", sql_table="t", data_source="ds").version == 8 def test_v7_payload_migrates_without_raising(self) -> None: - """``migrate()`` raises RuntimeError when no converter is - registered for a step, so v8_migration.py is mandatory, not optional.""" + """``migrate()`` raises if a step has no converter, so v8_migration.py is mandatory.""" payload = { "version": 7, "name": "orders", @@ -183,8 +177,7 @@ def test_v7_payload_migrates_without_raising(self) -> None: assert migrated["version"] == 8 def test_v7_payload_loads_with_unknown_source_kind(self) -> None: - """A pre-existing model cannot know what backed it — None is correct, - not a guess of "table".""" + """A pre-existing model cannot know what backed it — None, not a guess of "table".""" model = SlayerModel.model_validate( { "version": 7, @@ -210,15 +203,8 @@ def test_migration_chain_walks_from_v1(self) -> None: class TestSourceKindRefresh: async def test_view_to_table_flip_refreshes(self, workspace: Path) -> None: - """THE Part E regression. - - The replacement table has IDENTICAL columns on purpose: that produces - no new columns, no new joins and no widening, so it drives the early - return in ``_additive_merge_existing`` and the save gate in - ``_process_one_table``. An implementation that only adds source_kind to - ``model_copy(update=...)`` passes every other test in this file and - fails this one. - """ + """THE Part E regression: identical columns → no new columns/joins/widening, + so it drives the early return and the save gate, not just model_copy(update=...).""" db_path, ds = _ds( workspace, """ @@ -280,8 +266,7 @@ async def test_table_to_view_flip_refreshes(self, workspace: Path) -> None: assert second.source_kind == "view" async def test_refresh_reports_the_change(self, workspace: Path) -> None: - """a kind change with no column change must still surface as - a ModelAddition, otherwise the run looks like a silent no-op.""" + """a kind change with no column change must still surface as a ModelAddition.""" db_path, ds = _ds( workspace, """ @@ -304,8 +289,7 @@ async def test_refresh_reports_the_change(self, workspace: Path) -> None: assert addition.kind_change == "view → table" async def test_none_never_erases_a_known_value(self, workspace: Path) -> None: - """a fresh model from a path that does not classify (kind - None) must not wipe a persisted value.""" + """a fresh model with kind None must not wipe a persisted value.""" from slayer.engine.ingestion import _additive_merge_existing persisted = SlayerModel( @@ -327,8 +311,7 @@ async def test_none_never_erases_a_known_value(self, workspace: Path) -> None: assert outcome.merged.source_kind == "view" def test_no_op_merge_preserves_the_early_return_contract(self) -> None: - """an identical re-ingest must still short-circuit; the new - kind check must not make every merge look dirty.""" + """an identical re-ingest must still short-circuit; the kind check must not make every merge dirty.""" from slayer.engine.ingestion import _additive_merge_existing model = SlayerModel( @@ -350,8 +333,7 @@ def test_no_op_merge_preserves_the_early_return_contract(self) -> None: async def test_identical_reingest_is_still_a_no_op( self, workspace: Path ) -> None: - """End-to-end guard for the same thing: adding source_kind must not - turn every re-ingest into a write.""" + """End-to-end guard: adding source_kind must not turn every re-ingest into a write.""" _, ds = _ds(workspace, _TABLE_AND_VIEW) storage = YAMLStorage(base_dir=str(workspace / "storage")) await ingest_datasource_idempotent(datasource=ds, storage=storage) diff --git a/tests/test_v7_migration.py b/tests/test_v7_migration.py index fd60673a..09351a3a 100644 --- a/tests/test_v7_migration.py +++ b/tests/test_v7_migration.py @@ -28,9 +28,8 @@ def test_v7_step_is_registered() -> None: - """v8 exists, so this file no longer pins the *current* version (see - ``tests/test_model_source_kind.py``). What stays true is that the chain - reaches v7 and has a step out of it.""" + """v8 exists, so this pins only that the chain reaches v7 (the current + version is checked in ``tests/test_model_source_kind.py``).""" assert mig.CURRENT_VERSIONS["SlayerModel"] >= 7 assert ("SlayerModel", 6) in mig._REGISTRY @@ -240,8 +239,8 @@ async def test_sqlite_round_trips_v6_payload_to_v7_with_new_fields_none() -> Non def test_v5_payload_walks_through_chain_to_v7() -> None: """End-to-end migration from v5 through v6 to v7 via the orchestrator. - Asserts the current version rather than a literal 7: the point is that the - walk completes, not where it stops today.""" + Asserts the current version, not a literal 7 — the point is the walk + completes.""" raw = { "version": 5, "name": "orders", From ae9939c6d0d6bbf5c052e6bb9411514f5fcba341 Mon Sep 17 00:00:00 2001 From: Egor Kraev Date: Thu, 20 Aug 2026 12:03:41 +0200 Subject: [PATCH 6/7] fix: explicit default-schema re-ingest heals instead of false cross-schema conflict A bare ingest persists default-schema models unqualified; a later explicit --schema qualifies them (public.orders), which _cross_schema_conflict wrongly flagged as a conflict and skipped. Thread the default schema name through so a fresh object qualified with the default is treated as the qualifier repair it is. Also label the two new cli.md fenced blocks (markdownlint MD040). --- docs/reference/cli.md | 4 +- slayer/engine/ingestion.py | 15 ++++++++ tests/test_ingestion_schema_qualification.py | 40 ++++++++++++++++++++ 3 files changed, 57 insertions(+), 2 deletions(-) diff --git a/docs/reference/cli.md b/docs/reference/cli.md index 64b1cc2f..6ce6b277 100644 --- a/docs/reference/cli.md +++ b/docs/reference/cli.md @@ -113,7 +113,7 @@ With neither flag, ingest covers exactly one schema, resolved in this order: If other schemas exist, ingest names them and exits 0 — a hint, not a failure: -``` +```text Note: ingested schema 'main' only. Other schemas in this datasource: openfda_rest. Re-run with --schema openfda_rest, or --all-schemas, to ingest them. ``` @@ -131,7 +131,7 @@ verbatim, so `--schema public` keeps producing `public.orders`. A model whose `sql_table` is missing its schema qualifier is repaired on the next ingest of that schema, and the repair is reported: -``` +```text Updated: reports (sql_table: reports → openfda_rest.reports) ``` diff --git a/slayer/engine/ingestion.py b/slayer/engine/ingestion.py index 82dca30a..ee88ae3e 100644 --- a/slayer/engine/ingestion.py +++ b/slayer/engine/ingestion.py @@ -873,6 +873,10 @@ class IngestionScanReport(BaseModel): # unqualified model IS the default schema's table" from a same-named table # elsewhere. ``None`` (listing failed) is distinct from empty: fails closed. default_schema_objects: list[str] | None = None + # Bare default schema name, so the merge can tell a fresh object explicitly + # qualified with the default (``public.orders``) from one in another schema: + # the former heals an unqualified persisted model, it is not a conflict. + default_schema: str | None = None @property def hidden_internals(self) -> list[InternalTable]: @@ -1643,6 +1647,7 @@ def ingest_datasource_report( objects=discovered, include_views=include_views, ), + default_schema=_bare_schema(qualified_default_schema(inspector)) or None, internal_tables=internal_tables, ) finally: @@ -1912,6 +1917,7 @@ def _cross_schema_conflict( persisted: SlayerModel, fresh: SlayerModel, default_schema_objects: set[str] | None, + default_schema: str | None = None, ) -> SkippedTable | None: """Refuse to merge two different schemas' tables into one model. @@ -1936,6 +1942,12 @@ def _cross_schema_conflict( shadows_default = persisted_schema is None and ( default_schema_objects is None or persisted_table in default_schema_objects ) + # A fresh object qualified with the DEFAULT schema (explicit ``--schema + # public``) names the same table an unqualified persisted model does — a + # qualifier repair, not a conflict. A qualified persisted table in another + # schema still conflicts, so this only relaxes the shadows-default case. + if persisted_schema is None and fresh_schema == default_schema: + shadows_default = False if not (conflicting or shadows_default): return None return SkippedTable( @@ -1956,6 +1968,7 @@ async def _process_one_table( datasource: DatasourceConfig, storage: StorageBackend, default_schema_objects: set[str] | None = None, + default_schema: str | None = None, ) -> ProcessTableOutcome: """Save / merge one freshly-introspected model. Raises on persistence failure — the caller isolates errors per-model. @@ -1985,6 +1998,7 @@ async def _process_one_table( persisted=persisted, fresh=fresh, default_schema_objects=default_schema_objects, + default_schema=default_schema, ) if conflict is not None: return ProcessTableOutcome(skipped=conflict) @@ -2174,6 +2188,7 @@ async def ingest_datasource_idempotent( datasource=datasource, storage=storage, default_schema_objects=default_schema_objects, + default_schema=scan.default_schema, ) if outcome.addition is not None: additions.append(outcome.addition) diff --git a/tests/test_ingestion_schema_qualification.py b/tests/test_ingestion_schema_qualification.py index 7081171b..6fd45357 100644 --- a/tests/test_ingestion_schema_qualification.py +++ b/tests/test_ingestion_schema_qualification.py @@ -507,6 +507,24 @@ async def test_bare_persisted_model_is_not_repointed(self, tmp_path): result.skipped ) + async def test_explicit_default_schema_reingest_heals_not_conflicts( + self, tmp_path + ): + """Naming the default schema explicitly (``--schema main``) after a bare + ingest qualifies its objects, which must HEAL the unqualified persisted + model, not skip it as a cross-schema conflict.""" + ds = _collide_ds(tmp_path) + storage = _storage(tmp_path) + await _ingest(ds, storage) # bare -> sql_table: reports (unqualified) + + result = await _ingest(ds, storage, schemas=["main"]) + + saved = await storage.get_model("reports", data_source="ds") + assert saved.sql_table == "main.reports" + assert not any("cross-schema" in s.reason for s in result.skipped), ( + result.skipped + ) + # --------------------------------------------------------------------------- # 14-16. validate-models @@ -1586,6 +1604,28 @@ def test_a_known_empty_default_schema_still_allows_the_repair(self): default_schema_objects=set(), ) is None + def test_default_schema_qualified_fresh_is_a_repair_not_a_conflict(self): + """A fresh object qualified with the DEFAULT schema (explicit + ``--schema public``) heals an unqualified persisted model; a non-default + schema with the same bare name still conflicts.""" + persisted = SlayerModel( + name="reports", data_source="ds", sql_table="reports", + columns=[Column(name="a", type=DataType.INT)], + ) + fresh = SlayerModel( + name="reports", data_source="ds", sql_table="public.reports", + columns=[Column(name="a", type=DataType.INT)], + ) + assert _cross_schema_conflict( + model_name="reports", persisted=persisted, fresh=fresh, + default_schema_objects={"reports"}, default_schema="public", + ) is None + other = fresh.model_copy(update={"sql_table": "s2.reports"}) + assert _cross_schema_conflict( + model_name="reports", persisted=persisted, fresh=other, + default_schema_objects={"reports"}, default_schema="public", + ) is not None + class TestAdditiveMergeQualifierRules: """Pinned directly on ``_additive_merge_existing``; driving through ingestion From 73a9084badbed0f08038e22a93873d7d2c92ddda Mon Sep 17 00:00:00 2001 From: Egor Kraev Date: Fri, 21 Aug 2026 11:50:56 +0200 Subject: [PATCH 7/7] fix: match the default schema on the full discovery token, not the bare name The previous default-schema conflict relaxation compared bare schema names, so an ATTACHed DuckDB catalog's same-named 'main' (a different table) could bypass the cross-schema guard. Carry the catalog-qualified default token and match via _matches_default, so a bare 'main' still matches the default while 'attached.main' does not. Extends the regression test to cover the attached-catalog case. --- slayer/engine/ingestion.py | 36 ++++++++++++------- tests/test_ingestion_schema_qualification.py | 37 ++++++++++++-------- 2 files changed, 45 insertions(+), 28 deletions(-) diff --git a/slayer/engine/ingestion.py b/slayer/engine/ingestion.py index ee88ae3e..b8f98022 100644 --- a/slayer/engine/ingestion.py +++ b/slayer/engine/ingestion.py @@ -873,10 +873,12 @@ class IngestionScanReport(BaseModel): # unqualified model IS the default schema's table" from a same-named table # elsewhere. ``None`` (listing failed) is distinct from empty: fails closed. default_schema_objects: list[str] | None = None - # Bare default schema name, so the merge can tell a fresh object explicitly - # qualified with the default (``public.orders``) from one in another schema: - # the former heals an unqualified persisted model, it is not a conflict. - default_schema: str | None = None + # Default schema discovery TOKEN (catalog-qualified on DuckDB), so the merge + # can tell a fresh object explicitly qualified with the default + # (``public.orders``, ``main.orders``) from one in another schema — or an + # ATTACHed catalog's same-named ``main`` — where the former heals an + # unqualified persisted model and the latter is a genuine conflict. + default_schema_token: str | None = None @property def hidden_internals(self) -> list[InternalTable]: @@ -1647,7 +1649,7 @@ def ingest_datasource_report( objects=discovered, include_views=include_views, ), - default_schema=_bare_schema(qualified_default_schema(inspector)) or None, + default_schema_token=qualified_default_schema(inspector), internal_tables=internal_tables, ) finally: @@ -1917,7 +1919,7 @@ def _cross_schema_conflict( persisted: SlayerModel, fresh: SlayerModel, default_schema_objects: set[str] | None, - default_schema: str | None = None, + default_schema_token: str | None = None, ) -> SkippedTable | None: """Refuse to merge two different schemas' tables into one model. @@ -1943,10 +1945,18 @@ def _cross_schema_conflict( default_schema_objects is None or persisted_table in default_schema_objects ) # A fresh object qualified with the DEFAULT schema (explicit ``--schema - # public``) names the same table an unqualified persisted model does — a - # qualifier repair, not a conflict. A qualified persisted table in another - # schema still conflicts, so this only relaxes the shadows-default case. - if persisted_schema is None and fresh_schema == default_schema: + # public`` / ``--schema main``) names the same table an unqualified persisted + # model does — a qualifier repair, not a conflict. Matched on the full + # discovery token via ``_matches_default`` so a bare ``main`` still matches + # DuckDB's catalog-qualified default while an ATTACHed catalog's same-named + # ``main`` (a different table) does not. Only relaxes the shadows-default + # case; a qualified persisted table in another schema still conflicts. + fresh_schema_token = split_sql_table(fresh_table)[0] + if ( + persisted_schema is None + and fresh_schema_token is not None + and _matches_default(fresh_schema_token, default_schema_token) + ): shadows_default = False if not (conflicting or shadows_default): return None @@ -1968,7 +1978,7 @@ async def _process_one_table( datasource: DatasourceConfig, storage: StorageBackend, default_schema_objects: set[str] | None = None, - default_schema: str | None = None, + default_schema_token: str | None = None, ) -> ProcessTableOutcome: """Save / merge one freshly-introspected model. Raises on persistence failure — the caller isolates errors per-model. @@ -1998,7 +2008,7 @@ async def _process_one_table( persisted=persisted, fresh=fresh, default_schema_objects=default_schema_objects, - default_schema=default_schema, + default_schema_token=default_schema_token, ) if conflict is not None: return ProcessTableOutcome(skipped=conflict) @@ -2188,7 +2198,7 @@ async def ingest_datasource_idempotent( datasource=datasource, storage=storage, default_schema_objects=default_schema_objects, - default_schema=scan.default_schema, + default_schema_token=scan.default_schema_token, ) if outcome.addition is not None: additions.append(outcome.addition) diff --git a/tests/test_ingestion_schema_qualification.py b/tests/test_ingestion_schema_qualification.py index 6fd45357..dee070a4 100644 --- a/tests/test_ingestion_schema_qualification.py +++ b/tests/test_ingestion_schema_qualification.py @@ -1606,25 +1606,32 @@ def test_a_known_empty_default_schema_still_allows_the_repair(self): def test_default_schema_qualified_fresh_is_a_repair_not_a_conflict(self): """A fresh object qualified with the DEFAULT schema (explicit - ``--schema public``) heals an unqualified persisted model; a non-default - schema with the same bare name still conflicts.""" + ``--schema public`` / ``--schema main``) heals an unqualified persisted + model; a non-default schema — including an ATTACHed catalog's same-named + ``main`` — still conflicts.""" persisted = SlayerModel( name="reports", data_source="ds", sql_table="reports", columns=[Column(name="a", type=DataType.INT)], ) - fresh = SlayerModel( - name="reports", data_source="ds", sql_table="public.reports", - columns=[Column(name="a", type=DataType.INT)], - ) - assert _cross_schema_conflict( - model_name="reports", persisted=persisted, fresh=fresh, - default_schema_objects={"reports"}, default_schema="public", - ) is None - other = fresh.model_copy(update={"sql_table": "s2.reports"}) - assert _cross_schema_conflict( - model_name="reports", persisted=persisted, fresh=other, - default_schema_objects={"reports"}, default_schema="public", - ) is not None + + def _conflict(fresh_sql: str, token: str): + fresh = SlayerModel( + name="reports", data_source="ds", sql_table=fresh_sql, + columns=[Column(name="a", type=DataType.INT)], + ) + return _cross_schema_conflict( + model_name="reports", persisted=persisted, fresh=fresh, + default_schema_objects={"reports"}, default_schema_token=token, + ) + + # Explicit default schema → repair. + assert _conflict("public.reports", "public") is None + # DuckDB: bare ``main`` still matches the catalog-qualified default token. + assert _conflict("main.reports", "collide.main") is None + # A non-default schema still conflicts. + assert _conflict("s2.reports", "public") is not None + # An ATTACHed catalog's same-named ``main`` is a DIFFERENT table. + assert _conflict("attached.main.reports", "collide.main") is not None class TestAdditiveMergeQualifierRules: