diff --git a/datajunction-server/datajunction_server/api/semantic_layer.py b/datajunction-server/datajunction_server/api/semantic_layer.py index dc9737f4a..3da7cf259 100644 --- a/datajunction-server/datajunction_server/api/semantic_layer.py +++ b/datajunction-server/datajunction_server/api/semantic_layer.py @@ -10,13 +10,25 @@ """ import logging -from typing import Any +from collections.abc import Mapping +from typing import cast, Literal, TypeAlias from fastapi import Depends from fastapi.responses import JSONResponse -from pydantic import BaseModel, Field # pylint: disable=no-name-in-module +from pydantic import ( # pylint: disable=no-name-in-module + BaseModel, + ConfigDict, + Field, + JsonValue, + TypeAdapter, + ValidationError, +) from sqlalchemy.ext.asyncio import AsyncSession +from datajunction_server.construction.build_v3.types import ( + ColumnMetadata as GeneratedColumnMetadata, +) +from datajunction_server.database.column import Column from datajunction_server.database.node import Node, NodeRevision from datajunction_server.database.user import User from datajunction_server.errors import DJException @@ -26,6 +38,11 @@ generate_metrics_sql, ) from datajunction_server.models.node_type import NodeType +from datajunction_server.models.semantic_layer_metadata import ( + SEMANTIC_LAYER_METADATA_KEY, +) +from datajunction_server.models.unit import Unit, unit_to_dict +from datajunction_server.sql.parsing.types import ColumnType from datajunction_server.utils import get_current_user, get_session logger = logging.getLogger(__name__) @@ -69,6 +86,9 @@ "varchar": "utf8", } +_UNIT_ADAPTER = TypeAdapter(Unit) +JSONObject: TypeAlias = dict[str, JsonValue] + # --------------------------------------------------------------------------- # Helpers @@ -83,7 +103,7 @@ def _problem(status_code: int, detail: str) -> JSONResponse: ) -def _arrow_type_name(dj_type: Any) -> str | None: +def _arrow_type_name(dj_type: ColumnType | str | None) -> str | None: """Map a DJ column type to the Arrow JSON type object's ``name`` value.""" if not dj_type: return None @@ -102,21 +122,327 @@ def _cube_column_type_map(cube: NodeRevision) -> dict[str, str | None]: } -def _cube_metadata_map(cube: NodeRevision) -> dict[str, dict[str, str | None]]: - """Return column metadata for the metric/dimension ids exposed by a cube.""" - return { - column.cube_element_name: {"display_name": column.display_name} - for column in cube.columns +def _semantic_type( + column: Column, + arrow_type: str, + *, + is_metric: bool, + unit: Unit | None = None, +) -> str: + """Derive the portable business type from DJ's structured column fields.""" + if getattr(column, "name", "").rsplit(".", 1)[-1] == "dateint": + return "date" + + unit_data = unit_to_dict(unit or column.unit) + if unit_data and "kind" in unit_data: + unit_kind = unit_data["kind"] + if unit_kind == "time": + return "duration" + if unit_kind in { + "currency", + "percentage", + "proportion", + "count", + "data_size", + }: + return unit_kind + + attributes = set(column.attribute_names()) + if "primary_key" in attributes: + return "identifier" + if arrow_type == "date": + return "date" + if arrow_type == "timestamp": + return "timestamp" + if arrow_type == "bool": + return "boolean" + if arrow_type in {"int", "floating", "decimal"} or is_metric: + return "number" + return "category" + + +def _format_metadata(unit: Unit | None) -> "FormatMetadata | None": + """Translate units with unambiguous display semantics to portable hints.""" + unit_data = unit_to_dict(unit) + if not unit_data or "kind" not in unit_data: + return None + preset_by_kind = { + "currency": "currency", + "time": "duration", + "data_size": "data_size", } + preset = preset_by_kind.get(unit_data["kind"]) + return FormatMetadata(preset=preset) if preset else None + + +def _validated_unit(raw_unit: JsonValue) -> Unit | None: + """Validate custom metadata units without letting bad annotations fail a view.""" + if raw_unit is None: + return None + try: + return _UNIT_ADAPTER.validate_python(raw_unit) + except ValidationError: + return None + + +def _inferred_filter_metadata(arrow_type: str | None) -> "FilterMetadata | None": + """Infer conservative filter hints supported by DJ query translation.""" + if arrow_type in {"utf8", "bool"}: + operators = ["=", "!=", "IS NULL", "IS NOT NULL"] + kind = "text" if arrow_type == "utf8" else "boolean" + elif arrow_type in {"int", "floating", "decimal", "date", "timestamp"}: + operators = [ + "=", + "!=", + ">", + ">=", + "<", + "<=", + "IS NULL", + "IS NOT NULL", + ] + kind = { + "date": "date", + "timestamp": "datetime", + }.get(arrow_type, "number") + else: + return None + return FilterMetadata( + kind=kind, + operators=operators, + default_operator="=", + ) + + +def _decimal_places_pattern(precision: int) -> str: + """Build the decimal-place portion of a number format pattern.""" + return f".{''.join('0' for _ in range(precision))}" if precision else "" + + +def _fixed_decimal_pattern(prefix: str, precision: int) -> str: + """Build a Google Sheets fixed-decimal pattern.""" + return f"{prefix}#,##0{_decimal_places_pattern(precision)}" + + +def _client_format_extensions( + extensions: dict[str, JSONObject], + unit: Unit | None, + format_metadata: "FormatMetadata | None", + *, + format_is_explicit: bool, +) -> dict[str, JSONObject]: + """Fill missing client formats when portable semantics are unambiguous.""" + if format_metadata is None or format_metadata.preset is None: + return extensions + + preset = format_metadata.preset + precision = format_metadata.precision + if precision is None: + precision = 2 + + unit_data = unit_to_dict(unit) + is_usd = bool( + unit_data + and unit_data.get("kind") == "currency" + and unit_data.get("code") == "USD", + ) + + d3format: str | None = None + number_format: dict[str, str] | None = None + if preset == "currency" and is_usd: + d3format = f"$,.{precision}f" + number_format = { + "type": "CURRENCY", + "pattern": _fixed_decimal_pattern("$", precision), + } + elif format_is_explicit and preset == "number": + d3format = f",.{precision}f" + number_format = { + "type": "NUMBER", + "pattern": _fixed_decimal_pattern("", precision), + } + elif format_is_explicit and preset == "percentage": + d3format = f".{precision}%" + number_format = { + "type": "PERCENT", + "pattern": f"0{_decimal_places_pattern(precision)}%", + } + elif format_is_explicit and preset == "smart_number": + d3format = "SMART_NUMBER" + + if d3format is not None: + extensions.setdefault("superset", {}).setdefault("d3format", d3format) + if number_format is not None: + extensions.setdefault("google_sheets", {}).setdefault( + "numberFormat", + number_format, + ) + return extensions + + +_METADATA_KEYS = { + "display_name", + "semantic_type", + "unit", + "attributes", + "format", + "filter", + "extensions", +} + +def _raw_column_metadata(cube: NodeRevision, column_id: str) -> JSONObject: + """Find semantic metadata declared in an element node's custom metadata. -def _generated_column_arrow_type_name(column: Any) -> str: + Metric nodes describe one value, so their metadata may be declared directly. + Dimension nodes can expose several values and use a ``columns`` mapping. + The reserved ``semantic_layer`` key keeps this contract separate from other DJ + custom metadata. + """ + for element in getattr(cube, "cube_elements", []): + revision = element.node_revision + if revision is None: + continue + if revision.type == NodeType.METRIC and revision.name != column_id: + continue + if revision.type != NodeType.METRIC: + base_id = f"{revision.name}.{element.name}" + if column_id != base_id and not column_id.startswith(f"{base_id}["): + continue + + custom_metadata = revision.custom_metadata or {} + if not isinstance(custom_metadata, Mapping): + return {} + raw = custom_metadata.get(SEMANTIC_LAYER_METADATA_KEY, {}) + if revision.type != NodeType.METRIC and isinstance(raw, Mapping): + columns = raw.get("columns", {}) + raw = columns.get(element.name, {}) if isinstance(columns, Mapping) else {} + return cast(JSONObject, dict(raw)) if isinstance(raw, Mapping) else {} + return {} + + +def _column_metadata( + column: Column, + *, + is_metric: bool, + raw_metadata: Mapping[str, JsonValue] | None = None, +) -> "ColumnMetadata | None": + """Build strict portable metadata from a DJ cube column.""" + raw_metadata = raw_metadata or {} + mapped_arrow_type = _arrow_type_name(column.type) + arrow_type = mapped_arrow_type or ( + METRIC_FALLBACK_ARROW_TYPE_NAME + if is_metric + else DIMENSION_FALLBACK_ARROW_TYPE_NAME + ) + attributes = column.attribute_names() + extensions = ( + { + str(namespace): cast(JSONObject, dict(values)) + for namespace, values in raw_metadata.get("extensions", {}).items() + if isinstance(values, Mapping) + } + if isinstance(raw_metadata.get("extensions"), Mapping) + else {} + ) + producer_metadata = { + key: value for key, value in raw_metadata.items() if key not in _METADATA_KEYS + } + if producer_metadata: + extensions["datajunction"] = { + **producer_metadata, + **extensions.get("datajunction", {}), + } + + raw_format = raw_metadata.get("format") + format_metadata = None + if isinstance(raw_format, Mapping): + raw_preset = raw_format.get("preset") + format_metadata = FormatMetadata( + preset=( + raw_preset + if isinstance(raw_preset, str) + and raw_preset in FormatMetadata.allowed_presets() + else None + ), + precision=( + raw_format.get("precision") + if isinstance(raw_format.get("precision"), int) + and not isinstance(raw_format.get("precision"), bool) + and raw_format["precision"] >= 0 + else None + ), + scale=( + raw_format.get("scale") + if isinstance(raw_format.get("scale"), (int, float)) + and not isinstance(raw_format.get("scale"), bool) + else None + ), + ) + if not format_metadata.model_dump(exclude_none=True): + format_metadata = None + + raw_unit = raw_metadata.get("unit", unit_to_dict(column.unit)) + unit = _validated_unit(raw_unit) + resolved_format = format_metadata or _format_metadata(unit) + extensions = _client_format_extensions( + extensions, + unit, + resolved_format, + format_is_explicit=isinstance(raw_format, Mapping), + ) + raw_semantic_type = raw_metadata.get("semantic_type") + if "filter" in raw_metadata: + filter_metadata = ( + FilterMetadata.from_mapping(raw_metadata["filter"]) + if isinstance(raw_metadata.get("filter"), Mapping) + else None + ) + else: + filter_metadata = _inferred_filter_metadata(mapped_arrow_type) + metadata = ColumnMetadata( + display_name=( + raw_metadata.get("display_name") + if isinstance(raw_metadata.get("display_name"), str) + else column.display_name + ), + semantic_type=( + raw_semantic_type + if isinstance(raw_semantic_type, str) + and raw_semantic_type in ColumnMetadata.allowed_semantic_types() + else _semantic_type( + column, + arrow_type, + is_metric=is_metric, + unit=unit, + ) + ), + unit=unit, + attributes=( + list(dict.fromkeys(raw_metadata["attributes"])) + if isinstance(raw_metadata.get("attributes"), list) + and all(isinstance(value, str) for value in raw_metadata["attributes"]) + else attributes or None + ), + format=resolved_format, + filter=filter_metadata, + extensions=extensions or None, + ) + return metadata if metadata.model_dump(exclude_none=True) else None + + +def _cube_column_map(cube: NodeRevision) -> dict[str, Column]: + """Return role-aware cube columns keyed by semantic-layer id.""" + return {column.cube_element_name: column for column in cube.columns} + + +def _generated_column_arrow_type_name(column: GeneratedColumnMetadata) -> str: """Return the semantic-layer Arrow type name for a generated SQL column.""" - arrow_type = _arrow_type_name(getattr(column, "type", None)) + arrow_type = _arrow_type_name(column.type) if arrow_type: return arrow_type - semantic_type = str(getattr(column, "semantic_type", "") or "").lower() + semantic_type = column.semantic_type.lower() if semantic_type == "dimension": return DIMENSION_FALLBACK_ARROW_TYPE_NAME return METRIC_FALLBACK_ARROW_TYPE_NAME @@ -125,20 +451,23 @@ def _generated_column_arrow_type_name(column: Any) -> str: def _metrics_payload(cube: NodeRevision) -> list["MetricInfo"]: """Spec ``metrics`` list. ``definition`` is display-only.""" type_by_name = _cube_column_type_map(cube) - metadata_by_name = _cube_metadata_map(cube) + columns_by_name = _cube_column_map(cube) return [ MetricInfo( id=metric_name, name=metric_name.split(".")[-1], type=type_by_name.get(metric_name) or METRIC_FALLBACK_ARROW_TYPE_NAME, definition=metric_name, - description=None, + description=getattr(columns_by_name.get(metric_name), "description", None), aggregation="OTHER", - metadata=MetricsMetadata( - display_name=metadata_by_name.get(metric_name, {}).get( - "display_name", - "", - ), + metadata=( + _column_metadata( + columns_by_name[metric_name], + is_metric=True, + raw_metadata=_raw_column_metadata(cube, metric_name), + ) + if metric_name in columns_by_name + else None ), ) for metric_name in cube.cube_node_metrics @@ -148,17 +477,23 @@ def _metrics_payload(cube: NodeRevision) -> list["MetricInfo"]: def _dimensions_payload(cube: NodeRevision) -> list["DimensionInfo"]: """Spec ``dimensions`` list. Grain detection is deferred.""" type_by_name = _cube_column_type_map(cube) - metadata_by_name = _cube_metadata_map(cube) + columns_by_name = _cube_column_map(cube) return [ DimensionInfo( id=dim_ref, name=dim_ref.split(".")[-1], type=type_by_name.get(dim_ref) or DIMENSION_FALLBACK_ARROW_TYPE_NAME, definition=dim_ref, - description=None, + description=getattr(columns_by_name.get(dim_ref), "description", None), grain=None, - metadata=DimensionMetadata( - display_name=metadata_by_name.get(dim_ref, {}).get("display_name", ""), + metadata=( + _column_metadata( + columns_by_name[dim_ref], + is_metric=False, + raw_metadata=_raw_column_metadata(cube, dim_ref), + ) + if dim_ref in columns_by_name + else None ), ) for dim_ref in cube.cube_node_dimensions @@ -174,6 +509,7 @@ def _view_payload(cube: NodeRevision) -> "ViewDetail": """ return ViewDetail( name=cube.name, + display_name=cube.display_name, uid=cube.name, features=[], # no optional spec features for now dimensions=_dimensions_payload(cube), @@ -192,7 +528,7 @@ def _view_payload(cube: NodeRevision) -> "ViewDetail": _NULLARY_OPERATORS = frozenset({"IS NULL", "IS NOT NULL"}) -def _quote_value(value: Any) -> str: +def _quote_value(value: JsonValue) -> str: """Render a scalar Python value as a SQL literal via sqlglot, which handles quote-escaping and numeric/bool/NULL rendering for the scalars the client sends (strings, numbers, bools, null).""" @@ -241,7 +577,7 @@ class FilterPayload(BaseModel): type: str = "WHERE" column: str | None = None operator: str = "=" - value: Any = None + value: JsonValue = None class QueryPayload(BaseModel): @@ -268,10 +604,159 @@ class QueryRequest(BaseModel): # --------------------------------------------------------------------------- -class MetricsMetadata(BaseModel): - """Metric-specific field metadata""" +class FormatMetadata(BaseModel): + """Portable presentation hints from the semantic-layer specification.""" - display_name: str + preset: ( + Literal[ + "smart_number", + "number", + "currency", + "percentage", + "duration", + "data_size", + ] + | None + ) = None + precision: int | None = Field(default=None, ge=0) + scale: float | None = None + + model_config = ConfigDict(extra="forbid") + + @staticmethod + def allowed_presets() -> set[str]: + """Values accepted by the portable metadata contract.""" + return { + "smart_number", + "number", + "currency", + "percentage", + "duration", + "data_size", + } + + +class FilterMetadata(BaseModel): + """Portable query-builder hints from the semantic-layer specification.""" + + kind: ( + Literal[ + "text", + "number", + "range", + "date", + "datetime", + "boolean", + "select", + ] + | None + ) = None + operators: list[str] | None = None + default_operator: str | None = None + multi: bool | None = None + + model_config = ConfigDict(extra="forbid") + + @classmethod + def from_mapping(cls, raw: Mapping[str, JsonValue]) -> "FilterMetadata | None": + """Sanitize free-form DJ custom metadata into the strict contract.""" + allowed_operators = { + "=", + "!=", + ">", + ">=", + "<", + "<=", + "IN", + "NOT IN", + "IS NULL", + "IS NOT NULL", + "between", + "contains", + "starts_with", + "ends_with", + } + operators = raw.get("operators") + clean_operators = ( + list(dict.fromkeys(operators)) + if isinstance(operators, list) + and all( + isinstance(operator, str) and operator in allowed_operators + for operator in operators + ) + else None + ) + default_operator = raw.get("default_operator") + if ( + not isinstance(default_operator, str) + or default_operator not in allowed_operators + or (clean_operators is not None and default_operator not in clean_operators) + ): + default_operator = None + metadata = cls( + kind=( + raw.get("kind") + if raw.get("kind") + in {"text", "number", "range", "date", "datetime", "boolean", "select"} + else None + ), + operators=clean_operators, + default_operator=default_operator, + multi=raw.get("multi") if isinstance(raw.get("multi"), bool) else None, + ) + return metadata if metadata.model_dump(exclude_none=True) else None + + +class ColumnMetadata(BaseModel): + """Portable metadata shared by semantic dimensions and metrics.""" + + display_name: str | None = None + semantic_type: ( + Literal[ + "currency", + "percentage", + "proportion", + "count", + "duration", + "data_size", + "date", + "timestamp", + "identifier", + "category", + "url", + "boolean", + "number", + "string", + ] + | None + ) = None + unit: Unit | None = None + attributes: list[str] | None = None + format: FormatMetadata | None = None + filter: FilterMetadata | None = None + extensions: dict[str, JSONObject] | None = None + + model_config = ConfigDict(extra="forbid") + + @staticmethod + def allowed_semantic_types() -> set[str]: + """Values accepted by the portable metadata contract.""" + return { + "currency", + "percentage", + "proportion", + "count", + "duration", + "data_size", + "date", + "timestamp", + "identifier", + "category", + "url", + "boolean", + "number", + "string", + } class MetricInfo(BaseModel): @@ -283,13 +768,7 @@ class MetricInfo(BaseModel): definition: str description: str | None aggregation: str - metadata: MetricsMetadata - - -class DimensionMetadata(BaseModel): - """Dimension-specific field metadata""" - - display_name: str + metadata: ColumnMetadata | None = None class DimensionInfo(BaseModel): @@ -301,13 +780,14 @@ class DimensionInfo(BaseModel): definition: str description: str | None grain: str | None - metadata: DimensionMetadata + metadata: ColumnMetadata | None = None class ViewSummary(BaseModel): """Summary entry returned by ``/views/list`` (``{name, uid, features}``).""" name: str + display_name: str | None = None uid: str features: list[str] @@ -316,6 +796,7 @@ class ViewDetail(BaseModel): """Full semantic view returned by ``/views/{view}``.""" name: str + display_name: str | None = None uid: str features: list[str] dimensions: list[DimensionInfo] @@ -343,7 +824,11 @@ class GeneratedSQLResponse(BaseModel): # --------------------------------------------------------------------------- -@router.post("/views/list", response_model=list[ViewSummary]) +@router.post( + "/views/list", + response_model=list[ViewSummary], + response_model_exclude_none=True, +) async def list_views( session: AsyncSession = Depends(get_session), current_user: User = Depends(get_current_user), @@ -354,16 +839,28 @@ async def list_views( full metrics/dimensions are fetched per-view via ``/views/{view}``. """ try: - cube_names = await Node.find_names(session, node_type=NodeType.CUBE) + cubes = await Node.find_names_with_display_names( + session, + node_type=NodeType.CUBE, + ) except DJException as exc: return _problem(exc.http_status_code or 400, exc.message) return [ - ViewSummary(name=cube_name, uid=cube_name, features=[]) - for cube_name in cube_names + ViewSummary( + name=cube_name, + display_name=display_name, + uid=cube_name, + features=[], + ) + for cube_name, display_name in cubes ] -@router.post("/views/{view_name}", response_model=ViewDetail) +@router.post( + "/views/{view_name}", + response_model=ViewDetail, + response_model_exclude_none=True, +) async def get_view( view_name: str, session: AsyncSession = Depends(get_session), diff --git a/datajunction-server/datajunction_server/database/node.py b/datajunction-server/datajunction_server/database/node.py index 152738839..3dfc6113d 100644 --- a/datajunction-server/datajunction_server/database/node.py +++ b/datajunction-server/datajunction_server/database/node.py @@ -986,6 +986,28 @@ async def find_names( result = await session.execute(statement) return list(result.scalars().all()) + @classmethod + async def find_names_with_display_names( + cls, + session: AsyncSession, + prefix: str | None = None, + node_type: NodeType | None = None, + ) -> list[tuple[str, str | None]]: + """Find active node names and display names without ORM hydration.""" + statement = ( + select(Node.name, NodeRevision.display_name) + .join( + NodeRevision, + and_( + Node.id == NodeRevision.node_id, + Node.current_version == NodeRevision.version, + ), + ) + .where(*cls._find_filters(prefix, node_type)) + ) + result = await session.execute(statement) + return list(result.tuples().all()) + @classmethod async def main_branch_names( cls, diff --git a/datajunction-server/datajunction_server/internal/seed.py b/datajunction-server/datajunction_server/internal/seed.py index a92c160ab..2b60acbb0 100644 --- a/datajunction-server/datajunction_server/internal/seed.py +++ b/datajunction-server/datajunction_server/internal/seed.py @@ -4,17 +4,53 @@ import logging +from sqlalchemy import select from sqlalchemy.ext.asyncio import AsyncSession from datajunction_server.database.catalog import Catalog +from datajunction_server.database.custom_metadata_schema import CustomMetadataSchema from datajunction_server.database.engine import Engine from datajunction_server.models.dialect import Dialect +from datajunction_server.models.semantic_layer_metadata import ( + SEMANTIC_LAYER_METADATA_DESCRIPTION, + SEMANTIC_LAYER_METADATA_KEY, + SEMANTIC_LAYER_METADATA_SCHEMA, +) from datajunction_server.utils import get_settings logger = logging.getLogger(__name__) settings = get_settings() +async def seed_default_custom_metadata_schemas(session: AsyncSession) -> None: + """Register global schemas owned by the DJ server.""" + schema = ( + await session.execute( + select(CustomMetadataSchema).where( + CustomMetadataSchema.key == SEMANTIC_LAYER_METADATA_KEY, + CustomMetadataSchema.namespace.is_(None), + CustomMetadataSchema.node_type.is_(None), + CustomMetadataSchema.deactivated_at.is_(None), + ), + ) + ).scalar_one_or_none() + if schema is None: + schema = CustomMetadataSchema( + key=SEMANTIC_LAYER_METADATA_KEY, + namespace=None, + node_type=None, + ) + session.add(schema) + + schema.json_schema = SEMANTIC_LAYER_METADATA_SCHEMA + schema.value_kind = "object" + schema.filterable = False + schema.description = SEMANTIC_LAYER_METADATA_DESCRIPTION + schema.reserved = True + schema.deactivated_at = None + await session.commit() + + async def seed_default_catalogs(session: AsyncSession): """ Seeds two default catalogs: @@ -59,3 +95,5 @@ async def seed_default_catalogs(session: AsyncSession): await session.commit() logger.info("Added system catalog and engines") + + await seed_default_custom_metadata_schemas(session) diff --git a/datajunction-server/datajunction_server/models/semantic_layer_metadata.py b/datajunction-server/datajunction_server/models/semantic_layer_metadata.py new file mode 100644 index 000000000..db6be8cf5 --- /dev/null +++ b/datajunction-server/datajunction_server/models/semantic_layer_metadata.py @@ -0,0 +1,158 @@ +"""Built-in JSON Schema for semantic-layer custom metadata.""" + +SEMANTIC_LAYER_METADATA_KEY = "semantic_layer" +SEMANTIC_LAYER_METADATA_DESCRIPTION = ( + "Portable semantic-layer column metadata and client-specific extensions." +) + +_SEMANTIC_TYPES = [ + "currency", + "percentage", + "proportion", + "count", + "duration", + "data_size", + "date", + "timestamp", + "identifier", + "category", + "url", + "boolean", + "number", + "string", +] + +_FORMAT_PRESETS = [ + "smart_number", + "number", + "currency", + "percentage", + "duration", + "data_size", +] + +_FILTER_KINDS = [ + "text", + "number", + "range", + "date", + "datetime", + "boolean", + "select", +] + +_FILTER_OPERATORS = [ + "=", + "!=", + ">", + ">=", + "<", + "<=", + "IN", + "NOT IN", + "IS NULL", + "IS NOT NULL", + "between", + "contains", + "starts_with", + "ends_with", +] + +_COLUMN_METADATA_PROPERTIES = { + "display_name": {"type": "string"}, + "semantic_type": {"type": "string", "enum": _SEMANTIC_TYPES}, + "unit": {"$ref": "#/$defs/Unit"}, + "attributes": { + "type": "array", + "items": {"type": "string"}, + "uniqueItems": True, + }, + "format": {"$ref": "#/$defs/FormatMetadata"}, + "filter": {"$ref": "#/$defs/FilterMetadata"}, + "extensions": {"$ref": "#/$defs/Extensions"}, +} + +SEMANTIC_LAYER_METADATA_SCHEMA = { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$defs": { + "ColumnMetadata": { + "type": "object", + "additionalProperties": False, + "properties": _COLUMN_METADATA_PROPERTIES, + }, + "Unit": { + "oneOf": [ + {"$ref": "#/$defs/AtomicUnit"}, + {"$ref": "#/$defs/CompoundUnit"}, + ], + }, + "AtomicUnit": { + "type": "object", + "additionalProperties": False, + "required": ["kind"], + "properties": { + "kind": { + "type": "string", + "enum": [ + "currency", + "time", + "data_size", + "percentage", + "proportion", + "count", + "unitless", + ], + }, + "code": {"type": "string", "minLength": 1}, + }, + }, + "CompoundUnit": { + "type": "object", + "additionalProperties": False, + "required": ["numerator", "denominator"], + "properties": { + "numerator": {"$ref": "#/$defs/AtomicUnit"}, + "denominator": {"$ref": "#/$defs/AtomicUnit"}, + }, + }, + "FormatMetadata": { + "type": "object", + "additionalProperties": False, + "properties": { + "preset": {"type": "string", "enum": _FORMAT_PRESETS}, + "precision": {"type": "integer", "minimum": 0}, + "scale": {"type": "number"}, + }, + }, + "FilterMetadata": { + "type": "object", + "additionalProperties": False, + "properties": { + "kind": {"type": "string", "enum": _FILTER_KINDS}, + "operators": { + "type": "array", + "items": {"type": "string", "enum": _FILTER_OPERATORS}, + "uniqueItems": True, + }, + "default_operator": { + "type": "string", + "enum": _FILTER_OPERATORS, + }, + "multi": {"type": "boolean"}, + }, + }, + "Extensions": { + "type": "object", + "additionalProperties": {"type": "object"}, + }, + }, + "type": "object", + "additionalProperties": False, + "properties": { + **_COLUMN_METADATA_PROPERTIES, + "columns": { + "type": "object", + "additionalProperties": {"$ref": "#/$defs/ColumnMetadata"}, + }, + }, +} diff --git a/datajunction-server/tests/api/semantic_layer_test.py b/datajunction-server/tests/api/semantic_layer_test.py index b0dba2401..0ba10d67e 100644 --- a/datajunction-server/tests/api/semantic_layer_test.py +++ b/datajunction-server/tests/api/semantic_layer_test.py @@ -16,13 +16,16 @@ MAX_ROW_LIMIT, FilterPayload, _arrow_type_name, + _column_metadata, _dimensions_payload, _filter_to_sql, _generated_column_arrow_type_name, _metrics_payload, _quote_value, + _raw_column_metadata, ) from datajunction_server.errors import DJException +from datajunction_server.models.node_type import NodeType class TestFilterToSql: @@ -85,16 +88,25 @@ def test_metric_and_dimension_payloads_use_cube_column_types(self): cube_element_name="sem.total_amount", type="decimal(18,2)", display_name="Total amount", + description="Gross sales amount", + unit={"kind": "currency", "code": "USD"}, + attribute_names=lambda: ["certified"], ), SimpleNamespace( cube_element_name="sem.region.region_id", type="bigint", display_name="Region ID", + description=None, + unit=None, + attribute_names=lambda: ["primary_key"], ), SimpleNamespace( cube_element_name="sem.region.region_name[home]", type="varchar(255)", display_name="Home region name", + description=None, + unit=None, + attribute_names=lambda: [], ), ], cube_node_metrics=["sem.total_amount"], @@ -113,10 +125,42 @@ def test_metric_and_dimension_payloads_use_cube_column_types(self): "name": "total_amount", "type": "decimal", "definition": "sem.total_amount", - "description": None, + "description": "Gross sales amount", "aggregation": "OTHER", "metadata": { "display_name": "Total amount", + "semantic_type": "currency", + "unit": {"kind": "currency", "code": "USD"}, + "attributes": ["certified"], + "format": { + "preset": "currency", + "precision": None, + "scale": None, + }, + "filter": { + "kind": "number", + "operators": [ + "=", + "!=", + ">", + ">=", + "<", + "<=", + "IS NULL", + "IS NOT NULL", + ], + "default_operator": "=", + "multi": None, + }, + "extensions": { + "superset": {"d3format": "$,.2f"}, + "google_sheets": { + "numberFormat": { + "type": "CURRENCY", + "pattern": "$#,##0.00", + }, + }, + }, }, }, ] @@ -131,6 +175,26 @@ def test_metric_and_dimension_payloads_use_cube_column_types(self): "grain": None, "metadata": { "display_name": "Region ID", + "semantic_type": "identifier", + "unit": None, + "attributes": ["primary_key"], + "format": None, + "filter": { + "kind": "number", + "operators": [ + "=", + "!=", + ">", + ">=", + "<", + "<=", + "IS NULL", + "IS NOT NULL", + ], + "default_operator": "=", + "multi": None, + }, + "extensions": None, }, }, { @@ -142,6 +206,17 @@ def test_metric_and_dimension_payloads_use_cube_column_types(self): "grain": None, "metadata": { "display_name": "Home region name", + "semantic_type": "category", + "unit": None, + "attributes": None, + "format": None, + "filter": { + "kind": "text", + "operators": ["=", "!=", "IS NULL", "IS NOT NULL"], + "default_operator": "=", + "multi": None, + }, + "extensions": None, }, }, ] @@ -153,11 +228,15 @@ def test_metric_and_dimension_payloads_fallback_when_type_is_unknown(self): cube_element_name="sem.total_amount", type=None, display_name="Total amount", + unit=None, + attribute_names=lambda: [], ), SimpleNamespace( cube_element_name="sem.region.region_name", type="unknown_type", display_name="Region name", + unit=None, + attribute_names=lambda: [], ), ], cube_node_metrics=["sem.total_amount"], @@ -170,6 +249,441 @@ def test_metric_and_dimension_payloads_fallback_when_type_is_unknown(self): assert metrics[0].type == "floating" assert dimensions[0].type == "utf8" + def test_metric_metadata_reads_portable_custom_metadata(self): + metric_revision = SimpleNamespace( + name="sem.total_amount", + type=NodeType.METRIC, + custom_metadata={ + "semantic_layer": { + "display_name": "Net revenue", + "semantic_type": "currency", + "unit": {"kind": "currency", "code": "EUR"}, + "attributes": ["certified", "certified"], + "format": {"preset": "currency", "precision": 2}, + "filter": { + "kind": "range", + "operators": [">", ">="], + "default_operator": ">=", + }, + "extensions": { + "superset": {"d3format": "€,.2f"}, + "google_sheets": { + "numberFormat": { + "type": "CURRENCY", + "pattern": "€#,##0.00", + }, + }, + "datajunction": {"owner": "finance"}, + }, + }, + }, + ) + cube = SimpleNamespace( + columns=[ + SimpleNamespace( + cube_element_name="sem.total_amount", + type="double", + display_name="Total amount", + description=None, + unit=None, + attribute_names=lambda: [], + ), + ], + cube_elements=[SimpleNamespace(node_revision=metric_revision)], + cube_node_metrics=["sem.total_amount"], + ) + + metadata = _metrics_payload(cube)[0].metadata + + assert metadata is not None + assert metadata.model_dump(exclude_none=True) == { + "display_name": "Net revenue", + "semantic_type": "currency", + "unit": {"kind": "currency", "code": "EUR"}, + "attributes": ["certified"], + "format": {"preset": "currency", "precision": 2}, + "filter": { + "kind": "range", + "operators": [">", ">="], + "default_operator": ">=", + }, + "extensions": { + "superset": {"d3format": "€,.2f"}, + "google_sheets": { + "numberFormat": { + "type": "CURRENCY", + "pattern": "€#,##0.00", + }, + }, + "datajunction": {"owner": "finance"}, + }, + } + + def test_metric_metadata_ignores_custom_metadata_outside_reserved_key(self): + metric_revision = SimpleNamespace( + name="sem.total_amount", + type=NodeType.METRIC, + custom_metadata={ + "format": {"preset": "percentage"}, + "owner": "finance", + }, + ) + cube = SimpleNamespace( + cube_elements=[SimpleNamespace(node_revision=metric_revision)], + ) + + assert _raw_column_metadata(cube, "sem.total_amount") == {} + + @pytest.mark.parametrize( + ("format_metadata", "expected_extensions"), + [ + ( + {"preset": "number", "precision": 0}, + { + "superset": {"d3format": ",.0f"}, + "google_sheets": { + "numberFormat": {"type": "NUMBER", "pattern": "#,##0"}, + }, + }, + ), + ( + {"preset": "percentage"}, + { + "superset": {"d3format": ".2%"}, + "google_sheets": { + "numberFormat": {"type": "PERCENT", "pattern": "0.00%"}, + }, + }, + ), + ( + {"preset": "smart_number"}, + {"superset": {"d3format": "SMART_NUMBER"}}, + ), + ], + ) + def test_explicit_portable_formats_generate_client_fallbacks( + self, + format_metadata, + expected_extensions, + ): + column = SimpleNamespace( + type="double", + display_name="Value", + unit=None, + attribute_names=lambda: [], + ) + + metadata = _column_metadata( + column, + is_metric=True, + raw_metadata={"format": format_metadata}, + ) + + assert metadata is not None + assert metadata.extensions == expected_extensions + + def test_explicit_client_formats_take_precedence_over_generated_values(self): + column = SimpleNamespace( + type="double", + display_name="Revenue", + unit={"kind": "currency", "code": "USD"}, + attribute_names=lambda: [], + ) + + metadata = _column_metadata( + column, + is_metric=True, + raw_metadata={ + "format": {"preset": "currency", "precision": 0}, + "extensions": { + "superset": {"d3format": "EXPLICIT", "other": True}, + "google_sheets": { + "numberFormat": {"type": "TEXT"}, + "other": True, + }, + }, + }, + ) + + assert metadata is not None + assert metadata.extensions == { + "superset": {"d3format": "EXPLICIT", "other": True}, + "google_sheets": { + "numberFormat": {"type": "TEXT"}, + "other": True, + }, + } + + def test_explicit_usd_unit_generates_default_currency_formats(self): + column = SimpleNamespace( + type="double", + display_name="Revenue", + unit=None, + attribute_names=lambda: [], + ) + + metadata = _column_metadata( + column, + is_metric=True, + raw_metadata={"unit": {"kind": "currency", "code": "USD"}}, + ) + + assert metadata is not None + assert metadata.format is not None + assert metadata.format.preset == "currency" + assert metadata.extensions == { + "superset": {"d3format": "$,.2f"}, + "google_sheets": { + "numberFormat": { + "type": "CURRENCY", + "pattern": "$#,##0.00", + }, + }, + } + + def test_ambiguous_native_units_do_not_generate_client_formats(self): + percentage = SimpleNamespace( + type="double", + display_name="Percentage", + unit={"kind": "percentage"}, + attribute_names=lambda: [], + ) + euros = SimpleNamespace( + type="double", + display_name="Revenue", + unit={"kind": "currency", "code": "EUR"}, + attribute_names=lambda: [], + ) + + percentage_metadata = _column_metadata(percentage, is_metric=True) + euro_metadata = _column_metadata(euros, is_metric=True) + + assert percentage_metadata is not None + assert percentage_metadata.format is None + assert percentage_metadata.extensions is None + assert euro_metadata is not None + assert euro_metadata.format is not None + assert euro_metadata.format.preset == "currency" + assert euro_metadata.extensions is None + + @pytest.mark.parametrize( + ("column_type", "unit", "expected_semantic_type"), + [ + ("double", {"kind": "time", "code": "s"}, "duration"), + ("double", {"kind": "unitless"}, "number"), + ("date", None, "date"), + ("timestamp", None, "timestamp"), + ("boolean", None, "boolean"), + ], + ) + def test_semantic_type_inference_for_temporal_and_boolean_columns( + self, + column_type, + unit, + expected_semantic_type, + ): + column = SimpleNamespace( + type=column_type, + display_name="Value", + unit=unit, + attribute_names=lambda: [], + ) + + metadata = _column_metadata(column, is_metric=False) + + assert metadata is not None + assert metadata.semantic_type == expected_semantic_type + + @pytest.mark.parametrize( + "column_name", + ["dateint", "common.dimensions.time.date.dateint"], + ) + def test_dateint_columns_infer_date_semantic_type(self, column_name): + column = SimpleNamespace( + name=column_name, + type="int", + display_name="Date", + unit=None, + attribute_names=lambda: ["primary_key"], + ) + + metadata = _column_metadata(column, is_metric=False) + + assert metadata is not None + assert metadata.semantic_type == "date" + + @pytest.mark.parametrize( + ("column_type", "expected_kind", "expected_operators"), + [ + ("string", "text", ["=", "!=", "IS NULL", "IS NOT NULL"]), + ("boolean", "boolean", ["=", "!=", "IS NULL", "IS NOT NULL"]), + ( + "double", + "number", + ["=", "!=", ">", ">=", "<", "<=", "IS NULL", "IS NOT NULL"], + ), + ( + "date", + "date", + ["=", "!=", ">", ">=", "<", "<=", "IS NULL", "IS NOT NULL"], + ), + ( + "timestamp", + "datetime", + ["=", "!=", ">", ">=", "<", "<=", "IS NULL", "IS NOT NULL"], + ), + ], + ) + def test_filter_metadata_is_inferred_from_supported_column_types( + self, + column_type, + expected_kind, + expected_operators, + ): + column = SimpleNamespace( + type=column_type, + display_name="Value", + unit=None, + attribute_names=lambda: [], + ) + + metadata = _column_metadata(column, is_metric=False) + + assert metadata is not None + assert metadata.filter is not None + assert metadata.filter.kind == expected_kind + assert metadata.filter.operators == expected_operators + assert metadata.filter.default_operator == "=" + assert metadata.filter.multi is None + + @pytest.mark.parametrize( + "column_type", + ["array", "map", "struct", "binary", "time", "unknown_type"], + ) + def test_filter_metadata_is_not_inferred_for_unsupported_types( + self, + column_type, + ): + column = SimpleNamespace( + type=column_type, + display_name="Value", + unit=None, + attribute_names=lambda: [], + ) + + metadata = _column_metadata(column, is_metric=False) + + assert metadata is not None + assert metadata.filter is None + + @pytest.mark.parametrize( + ("explicit_filter", "expected_filter"), + [ + ( + { + "kind": "select", + "operators": ["IN", "NOT IN"], + "default_operator": "IN", + "multi": True, + }, + { + "kind": "select", + "operators": ["IN", "NOT IN"], + "default_operator": "IN", + "multi": True, + }, + ), + (None, None), + ], + ) + def test_explicit_filter_metadata_overrides_or_disables_inference( + self, + explicit_filter, + expected_filter, + ): + column = SimpleNamespace( + type="string", + display_name="Value", + unit=None, + attribute_names=lambda: [], + ) + + metadata = _column_metadata( + column, + is_metric=False, + raw_metadata={"filter": explicit_filter}, + ) + + assert metadata is not None + assert ( + metadata.filter.model_dump(exclude_none=True) + if metadata.filter is not None + else None + ) == expected_filter + + def test_invalid_optional_metadata_is_ignored(self): + column = SimpleNamespace( + type="double", + display_name="Value", + unit=None, + attribute_names=lambda: [], + ) + + metadata = _column_metadata( + column, + is_metric=True, + raw_metadata={ + "unit": {"kind": "currency", "code": "usd"}, + "format": {}, + "filter": { + "kind": "number", + "operators": ["="], + "default_operator": ">", + }, + }, + ) + + assert metadata is not None + assert metadata.unit is None + assert metadata.format is None + assert metadata.filter is not None + assert metadata.filter.default_operator is None + + def test_dimension_metadata_skips_unusable_elements_and_reads_wrapper(self): + cube = SimpleNamespace( + cube_elements=[ + SimpleNamespace(node_revision=None), + SimpleNamespace( + name="other", + node_revision=SimpleNamespace( + name="sem.region", + type=NodeType.DIMENSION, + custom_metadata={}, + ), + ), + SimpleNamespace( + name="region_name", + node_revision=SimpleNamespace( + name="sem.region", + type=NodeType.DIMENSION, + custom_metadata={ + "semantic_layer": { + "columns": { + "region_name": { + "display_name": "Sales region", + }, + }, + }, + }, + ), + ), + ], + ) + + metadata = _raw_column_metadata(cube, "sem.region.region_name") + + assert metadata == {"display_name": "Sales region"} + def test_metric_and_dimension_payloads_fallback_when_column_is_missing(self): cube = SimpleNamespace( columns=[], @@ -318,6 +832,22 @@ async def _setup_cube(client: AsyncClient) -> str: "description": "", "mode": "published", "query": "SELECT SUM(amount) FROM sem.sales", + "custom_metadata": { + "semantic_layer": { + "semantic_type": "currency", + "unit": {"kind": "currency", "code": "USD"}, + "format": {"preset": "currency", "precision": 2}, + "extensions": { + "superset": {"d3format": "$,.2f"}, + "google_sheets": { + "numberFormat": { + "type": "CURRENCY", + "pattern": "$#,##0.00", + }, + }, + }, + }, + }, }, ), 200, @@ -357,7 +887,8 @@ async def test_semantic_endpoints_end_to_end(client: AsyncClient): await client.post("/semantic/views/list", json={"runtime_configuration": {}}), 200, ) - assert any(v["name"] == view for v in resp.json()) + view_summary = next(v for v in resp.json() if v["name"] == view) + assert view_summary["display_name"] == "Sales Cube" # /views/{view} returns the cube's metrics and dimensions in spec shape. resp = await _expect( @@ -368,8 +899,50 @@ async def test_semantic_endpoints_end_to_end(client: AsyncClient): 200, ) detail = resp.json() + assert detail["display_name"] == "Sales Cube" assert {m["id"] for m in detail["metrics"]} == {"sem.total_amount"} assert any(d["id"] == "sem.region.region_name" for d in detail["dimensions"]) + assert detail["metrics"][0]["metadata"] == { + "display_name": "Total Amount", + "semantic_type": "currency", + "unit": {"kind": "currency", "code": "USD"}, + "format": {"preset": "currency", "precision": 2}, + "filter": { + "kind": "number", + "operators": [ + "=", + "!=", + ">", + ">=", + "<", + "<=", + "IS NULL", + "IS NOT NULL", + ], + "default_operator": "=", + }, + "extensions": { + "superset": {"d3format": "$,.2f"}, + "google_sheets": { + "numberFormat": { + "type": "CURRENCY", + "pattern": "$#,##0.00", + }, + }, + }, + } + dimension = next( + dim for dim in detail["dimensions"] if dim["id"] == "sem.region.region_name" + ) + assert dimension["metadata"] == { + "display_name": "Region Name", + "semantic_type": "category", + "filter": { + "kind": "text", + "operators": ["=", "!=", "IS NULL", "IS NOT NULL"], + "default_operator": "=", + }, + } # /sql generates physical SQL, pinned to this cube, in the trino dialect. resp = await _expect( @@ -516,7 +1089,7 @@ async def test_list_views_djexception_returns_problem( ): """``Node.find_names`` raising DJException -> problem response in list_views.""" monkeypatch.setattr( - "datajunction_server.api.semantic_layer.Node.find_names", + "datajunction_server.api.semantic_layer.Node.find_names_with_display_names", AsyncMock( side_effect=DJException(message="find blew up", http_status_code=418), ), diff --git a/datajunction-server/tests/internal/seed_test.py b/datajunction-server/tests/internal/seed_test.py index 1b6185d9d..64f1a611e 100644 --- a/datajunction-server/tests/internal/seed_test.py +++ b/datajunction-server/tests/internal/seed_test.py @@ -1,9 +1,22 @@ import pytest +from sqlalchemy import select from sqlalchemy.ext.asyncio import AsyncSession from datajunction_server.database.catalog import Catalog +from datajunction_server.database.custom_metadata_schema import CustomMetadataSchema from datajunction_server.database.engine import Engine -from datajunction_server.internal.seed import seed_default_catalogs +from datajunction_server.errors import DJInvalidInputException +from datajunction_server.internal.custom_metadata import validate_custom_metadata +from datajunction_server.internal.seed import ( + seed_default_catalogs, + seed_default_custom_metadata_schemas, +) +from datajunction_server.models.semantic_layer_metadata import ( + SEMANTIC_LAYER_METADATA_DESCRIPTION, + SEMANTIC_LAYER_METADATA_KEY, + SEMANTIC_LAYER_METADATA_SCHEMA, +) +from datajunction_server.models.node_type import NodeType from datajunction_server.utils import get_settings @@ -122,3 +135,79 @@ async def test_seed_default_catalogs_system_exists(clean_session: AsyncSession): ], ) assert len(catalogs) == 2 + + +@pytest.mark.asyncio +async def test_seed_registers_reserved_semantic_layer_schema( + clean_session: AsyncSession, +): + await seed_default_custom_metadata_schemas(clean_session) + + schema = ( + await clean_session.execute( + select(CustomMetadataSchema).where( + CustomMetadataSchema.key == SEMANTIC_LAYER_METADATA_KEY, + ), + ) + ).scalar_one() + assert schema.namespace is None + assert schema.node_type is None + assert schema.json_schema == SEMANTIC_LAYER_METADATA_SCHEMA + assert schema.value_kind == "object" + assert schema.filterable is False + assert schema.description == SEMANTIC_LAYER_METADATA_DESCRIPTION + assert schema.reserved is True + + +@pytest.mark.asyncio +async def test_seed_semantic_layer_schema_is_idempotent(clean_session: AsyncSession): + await seed_default_custom_metadata_schemas(clean_session) + await seed_default_custom_metadata_schemas(clean_session) + + schemas = ( + ( + await clean_session.execute( + select(CustomMetadataSchema).where( + CustomMetadataSchema.key == SEMANTIC_LAYER_METADATA_KEY, + CustomMetadataSchema.namespace.is_(None), + CustomMetadataSchema.node_type.is_(None), + ), + ) + ) + .scalars() + .all() + ) + assert len(schemas) == 1 + + +@pytest.mark.asyncio +async def test_seeded_semantic_layer_schema_validates_node_writes( + clean_session: AsyncSession, +): + await seed_default_custom_metadata_schemas(clean_session) + + await validate_custom_metadata( + clean_session, + "finance", + NodeType.METRIC, + { + "semantic_layer": { + "semantic_type": "currency", + "extensions": {"superset": {"d3format": "$,.2f"}}, + }, + }, + ) + with pytest.raises(DJInvalidInputException, match="sematic_type"): + await validate_custom_metadata( + clean_session, + "finance", + NodeType.METRIC, + {"semantic_layer": {"sematic_type": "currency"}}, + ) + with pytest.raises(DJInvalidInputException, match="owner"): + await validate_custom_metadata( + clean_session, + "finance", + NodeType.METRIC, + {"semantic_layer": {"owner": "finance"}}, + ) diff --git a/datajunction-server/tests/internal/test_custom_metadata_resolution.py b/datajunction-server/tests/internal/test_custom_metadata_resolution.py index 2ce1f576d..7c5b5a091 100644 --- a/datajunction-server/tests/internal/test_custom_metadata_resolution.py +++ b/datajunction-server/tests/internal/test_custom_metadata_resolution.py @@ -6,6 +6,10 @@ from datajunction_server.database.custom_metadata_schema import CustomMetadataSchema from datajunction_server.internal.custom_metadata import resolve_schemas from datajunction_server.models.node_type import NodeType +from datajunction_server.models.semantic_layer_metadata import ( + SEMANTIC_LAYER_METADATA_KEY, + SEMANTIC_LAYER_METADATA_SCHEMA, +) @pytest.mark.asyncio @@ -167,14 +171,16 @@ async def test_deactivated_rows_are_excluded(session): @pytest.mark.asyncio -async def test_empty_registry_returns_empty_dict(session): - """No schemas registered → empty result.""" +async def test_registry_with_only_builtins_returns_semantic_layer_schema(session): + """The startup seed makes the global semantic-layer schema always applicable.""" resolved = await resolve_schemas( session, namespace="any.namespace", node_type=NodeType.METRIC, ) - assert resolved == {} + assert resolved == { + SEMANTIC_LAYER_METADATA_KEY: SEMANTIC_LAYER_METADATA_SCHEMA, + } @pytest.mark.asyncio @@ -196,7 +202,9 @@ async def test_namespace_scoped_row_does_not_apply_to_none_namespace(session): namespace=None, node_type=NodeType.METRIC, ) - assert resolved == {} + assert resolved == { + SEMANTIC_LAYER_METADATA_KEY: SEMANTIC_LAYER_METADATA_SCHEMA, + } @pytest.mark.asyncio diff --git a/datajunction-server/tests/internal/test_custom_metadata_validation.py b/datajunction-server/tests/internal/test_custom_metadata_validation.py index ce584737c..c217862f0 100644 --- a/datajunction-server/tests/internal/test_custom_metadata_validation.py +++ b/datajunction-server/tests/internal/test_custom_metadata_validation.py @@ -1,5 +1,7 @@ """Tests for custom_metadata write-time validation helper.""" +from unittest.mock import AsyncMock + import pytest from datajunction_server.database.custom_metadata_schema import CustomMetadataSchema @@ -55,6 +57,25 @@ async def test_unregistered_key_passes(session): ) # no raise +@pytest.mark.asyncio +async def test_nonempty_metadata_passes_when_registry_is_empty(session, monkeypatch): + """Validation remains a no-op when startup schemas have not been seeded.""" + resolve_schemas = AsyncMock(return_value={}) + monkeypatch.setattr( + "datajunction_server.internal.custom_metadata.resolve_schemas", + resolve_schemas, + ) + + await validate_custom_metadata( + session, + None, + NodeType.METRIC, + {"anything": {"nested": True}}, + ) + + resolve_schemas.assert_awaited_once_with(session, None, NodeType.METRIC) + + @pytest.mark.asyncio async def test_empty_and_none_pass(session): await validate_custom_metadata(session, None, NodeType.METRIC, None) diff --git a/datajunction-server/tests/models/semantic_layer_metadata_test.py b/datajunction-server/tests/models/semantic_layer_metadata_test.py new file mode 100644 index 000000000..f08caa1c5 --- /dev/null +++ b/datajunction-server/tests/models/semantic_layer_metadata_test.py @@ -0,0 +1,68 @@ +"""Tests for the built-in semantic-layer custom metadata schema.""" + +import jsonschema +import pytest + +from datajunction_server.models.semantic_layer_metadata import ( + SEMANTIC_LAYER_METADATA_SCHEMA, +) + + +def test_semantic_layer_metadata_schema_is_valid_json_schema(): + jsonschema.Draft202012Validator.check_schema(SEMANTIC_LAYER_METADATA_SCHEMA) + + +@pytest.mark.parametrize( + "metadata", + [ + { + "display_name": "Net revenue", + "semantic_type": "currency", + "unit": {"kind": "currency", "code": "USD"}, + "format": {"preset": "currency", "precision": 2}, + "filter": { + "kind": "range", + "operators": ["=", ">", ">="], + "default_operator": ">=", + }, + "extensions": { + "superset": {"d3format": "$,.2f"}, + "google_sheets": { + "numberFormat": { + "type": "CURRENCY", + "pattern": "$#,##0.00", + }, + }, + }, + }, + { + "columns": { + "country": { + "semantic_type": "category", + "filter": { + "kind": "select", + "operators": ["IN", "NOT IN"], + "default_operator": "IN", + "multi": True, + }, + }, + }, + }, + ], +) +def test_semantic_layer_metadata_schema_accepts_supported_shapes(metadata): + jsonschema.validate(metadata, SEMANTIC_LAYER_METADATA_SCHEMA) + + +@pytest.mark.parametrize( + "metadata", + [ + {"sematic_type": "currency"}, + {"format": {"preset": "money"}}, + {"extensions": {"superset": "not-an-object"}}, + {"columns": {"country": {"filter": {"kind": "dropdown"}}}}, + ], +) +def test_semantic_layer_metadata_schema_rejects_invalid_shapes(metadata): + with pytest.raises(jsonschema.ValidationError): + jsonschema.validate(metadata, SEMANTIC_LAYER_METADATA_SCHEMA) diff --git a/docs/content/0.1.0/docs/data-modeling/custom-metadata.md b/docs/content/0.1.0/docs/data-modeling/custom-metadata.md index 0486add7f..88b860276 100644 --- a/docs/content/0.1.0/docs/data-modeling/custom-metadata.md +++ b/docs/content/0.1.0/docs/data-modeling/custom-metadata.md @@ -58,6 +58,44 @@ curl -X POST $DJ_SERVER/metadata-schemas/ \ If a repo manages the namespace, this is refused. A deployment reconciles that namespace to exactly what its manifest declares, so anything you register here would be undone on the next push. The error tells you to declare it in the repo instead. +### Built-in semantic-layer schema + +DJ automatically registers `semantic_layer` as a global, reserved metadata key. +It validates the portable metadata returned by the semantic-layer API, including +units, formats, filter hints, and namespaced client extensions. You do not need to +declare this schema in individual deployment manifests. + +Metric nodes put column metadata directly beneath the key: + +```yaml +custom_metadata: + semantic_layer: + format: + preset: currency + precision: 2 + extensions: + superset: + d3format: "$,.2f" +``` + +Dimension nodes declare metadata by column: + +```yaml +custom_metadata: + semantic_layer: + columns: + country: + filter: + kind: select + operators: [IN, NOT IN] + default_operator: IN + multi: true +``` + +The key is reserved so namespace-scoped schemas cannot redefine the shared +contract. Producer-specific fields must be placed inside an object under +`extensions`. + ## What validation does and does not do Validation is **lax about keys it does not know**. A key with no registered schema passes untouched. That means registering your first schema breaks nothing that already exists, and the feature is inert until you opt into it.