Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 4 additions & 5 deletions datasets/graphql/types.py
Original file line number Diff line number Diff line change
Expand Up @@ -516,16 +516,15 @@ def validation_violations(root: 'DatasetType') -> "list['DatasetValidationViolat
return []
from datasets.validation import load_violations
from nodes.dataset_materialization import ensure_dataset_materializations
from nodes.graphql.types.problems import DatasetValidationViolationType
from nodes.graphql.types.problems import DatasetValidationViolationType, build_coordinate_labels

materializations = ensure_dataset_materializations([root._model])
materialization = materializations.get(root._model.pk)
if materialization is None:
return []
return [
DatasetValidationViolationType.from_violation(violation)
for violation in load_violations(materialization.validation_violations)
]
violations = load_violations(materialization.validation_violations)
labels = build_coordinate_labels(violations)
return [DatasetValidationViolationType.from_violation(violation, labels) for violation in violations]

@sb.field(graphql_type=list[Annotated['DatasetPortType', sb.lazy('nodes.graphql.types.graph')]])
@staticmethod
Expand Down
8 changes: 4 additions & 4 deletions nodes/graphql/types/instance.py
Original file line number Diff line number Diff line change
Expand Up @@ -338,11 +338,11 @@ def constraint_conflicts(root: 'InstanceEditorFields', info: gql.Info) -> list[C
@staticmethod
def dataset_validation_violations(root: 'InstanceEditorFields') -> list[DatasetValidationViolationType]:
from nodes.dataset_materialization import collect_instance_dataset_violations
from nodes.graphql.types.problems import build_coordinate_labels

return [
DatasetValidationViolationType.from_violation(violation)
for violation in collect_instance_dataset_violations(root._config)
]
violations = collect_instance_dataset_violations(root._config)
labels = build_coordinate_labels(violations)
return [DatasetValidationViolationType.from_violation(violation, labels) for violation in violations]

@sb.field(
graphql_type=list[InstanceProblemInterface],
Expand Down
92 changes: 86 additions & 6 deletions nodes/graphql/types/problems.py
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,76 @@ class InstanceProblemInterface:
class DatasetDimensionCoordinateType:
dimension: str = sb.field(description='Dimension column identifier in the dataset.')
category: str = sb.field(description='Category identifier within the dimension.')
dimension_label: str = sb.field(
description="The dimension's label in the active language; falls back to the identifier when unresolvable."
)
category_label: str = sb.field(
description="The category's label in the active language; falls back to the identifier when unresolvable."
)


#: ``(dataset_uuid, dimension column, category identifier) -> (dimension label, category label)``.
type CoordinateLabels = dict[tuple[UUID | None, str, str], tuple[str, str]]


def build_coordinate_labels(violations: Iterable[RuleViolation]) -> CoordinateLabels:
"""
Resolve localized labels for the dimension coordinates named by ``violations``.

Violations are persisted with identifiers only: a materialization is shared
across users and languages, and ``RuleViolation.key`` diffs on the identifier
coordinates at edit time. Labels are therefore a presentation concern resolved
here, in the active language, rather than baked into the stored payload.

One query pass over the datasets involved, so a violation list costs a fixed
number of queries rather than one per coordinate.
"""
from kausal_common.datasets.models import Dataset, DatasetSchemaDimension, DimensionScope

dataset_uuids = {violation.dataset_uuid for violation in violations if violation.dataset_uuid is not None}
if not dataset_uuids:
return {}
datasets = list(
Dataset.objects
.filter(uuid__in=dataset_uuids)
.select_related('schema')
.only('uuid', 'schema', 'scope_content_type', 'scope_id')
)
labels: CoordinateLabels = {}
for dataset in datasets:
schema = dataset.schema
if schema is None or dataset.scope_id is None:
continue
# The dataframe column is DatasetSchemaDimension.column_name when set and the
# scoped dimension identifier otherwise -- the same rule the evaluator applies
# when it records the coordinate.
scopes = {
scope.dimension_id: scope
for scope in DimensionScope.objects
.filter(
scope_content_type=dataset.scope_content_type,
scope_id=dataset.scope_id,
Comment on lines +87 to +91

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P2 Badge Batch label queries across datasets

For an instance-wide violation list spanning multiple datasets, this queryset is executed once per dataset by the enclosing loop, its category prefetch adds another query per dataset, and the later DatasetSchemaDimension lookup adds a third. Thus the advertised fixed query cost is actually at least 3N + 1 for N affected datasets, which can make the editor's problem-list resolver issue dozens of queries; load scopes, categories, and schema dimensions in bulk for all collected datasets.

Useful? React with 👍 / 👎.

dimension_id__in=schema.dimensions.values_list('dimension_id', flat=True),
)
.select_related('dimension')
.prefetch_related('dimension__categories')
}
for schema_dimension in DatasetSchemaDimension.objects.filter(schema=schema).only('dimension_id', 'column_name'):
scope = scopes.get(schema_dimension.dimension_id)
if scope is None:
continue
column = schema_dimension.column_name or scope.identifier
if not column:
continue
dimension_label = scope.dimension.name_i18n or column
for category in scope.dimension.categories.all():
if category.identifier is None:
continue
labels[(dataset.uuid, column, category.identifier)] = (
dimension_label,
category.label_i18n or category.identifier,
)
return labels


@sb.type(
Expand All @@ -59,7 +129,18 @@ class DatasetValidationViolationType(InstanceProblemInterface):
requirement_group: str | None = sb.field(description='Named required-combination group, when applicable.')

@classmethod
def from_violation(cls, violation: RuleViolation) -> Self:
def from_violation(cls, violation: RuleViolation, labels: CoordinateLabels | None = None) -> Self:
labels = labels if labels is not None else {}
Comment on lines +132 to +133

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P2 Badge Populate labels for every violation construction path

When clients request the new fields through InstanceEditorFields.problems or any data-point mutation returning DatasetEditorMutation._current_violations, those unchanged call sites still invoke from_violation(violation) without a lookup. This optional default therefore silently returns the dimension and category identifiers as labels even when localized labels are resolvable, making the same GraphQL type behave differently depending on which field or mutation produced it; build and pass the lookup in those paths as well, or make omission impossible.

AGENTS.md reference: AGENTS.md:L221-L221

Useful? React with 👍 / 👎.


def coordinate(dimension: str, category: str) -> DatasetDimensionCoordinateType:
dimension_label, category_label = labels.get((violation.dataset_uuid, dimension, category), (dimension, category))
return DatasetDimensionCoordinateType(
dimension=dimension,
category=category,
dimension_label=dimension_label,
category_label=category_label,
)

return cls(
code=violation.kind,
message=violation.message,
Expand All @@ -71,10 +152,7 @@ def from_violation(cls, violation: RuleViolation) -> Self:
dataset_uuid=violation.dataset_uuid,
dataset_identifier=violation.dataset_identifier,
years=list(violation.years),
coordinates=[
DatasetDimensionCoordinateType(dimension=dimension, category=category)
for dimension, category in violation.categories.items()
],
coordinates=[coordinate(dimension, category) for dimension, category in violation.categories.items()],
combination_ids=list(violation.combination_ids),
requirement_group=violation.requirement_group,
)
Expand All @@ -92,4 +170,6 @@ class DatasetValidationViolationsType:

@classmethod
def from_violations(cls, violations: Iterable[RuleViolation]) -> Self:
return cls(violations=[DatasetValidationViolationType.from_violation(violation) for violation in violations])
found = list(violations)
labels = build_coordinate_labels(found)
return cls(violations=[DatasetValidationViolationType.from_violation(violation, labels) for violation in found])