Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions blueprint/builder.py
Original file line number Diff line number Diff line change
Expand Up @@ -79,9 +79,9 @@ def _config_to_params(
"""
from airflow.models.param import Param

from blueprint.core import _resolve_refs
from blueprint.core import _collapse_nullable, _resolve_refs

schema = _resolve_refs(config_type.model_json_schema())
schema = _collapse_nullable(_resolve_refs(config_type.model_json_schema()))
properties = schema.get("properties", {})
params: dict[str, Any] = {}

Expand Down
74 changes: 72 additions & 2 deletions blueprint/core.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
import copy
import inspect
import re
from collections.abc import Callable
from typing import (
TYPE_CHECKING,
Any,
Expand Down Expand Up @@ -99,6 +100,75 @@ def _is_yaml_compatible(annotation: Any, seen: set[int] | None = None) -> str |
return f"type {annotation!r} is not YAML-compatible"


_NULL_SCHEMA = {"type": "null"}


def _rewrite_nullable(schema: dict, rewrite: Callable[[dict, dict], dict]) -> dict:
"""Apply ``rewrite(field_keys, branch)`` to every ``X | None`` property in a schema.

``str | None`` reaches JSON Schema as ``anyOf: [{type: string}, {type: null}]``.
Only a single typed non-null branch is rewritten; anything else is left alone.
"""

def _walk(node: Any) -> Any:
if isinstance(node, list):
return [_walk(item) for item in node]
if not isinstance(node, dict):
return node

node = {key: _walk(value) for key, value in node.items()}
variants = node.get("anyOf")
if not isinstance(variants, list) or _NULL_SCHEMA not in variants:
return node

branches = [v for v in variants if v != _NULL_SCHEMA]
if len(branches) != 1 or not isinstance(branches[0].get("type"), str):
return node

# Field-level keys win: a nested model's title/description must not displace
# the ones the field itself declares.
field_keys = {key: value for key, value in node.items() if key != "anyOf"}
return rewrite(field_keys, branches[0])

return _walk(schema)


def _strip_nullable(schema: dict) -> dict:
"""Reduce ``X | None`` to a plain ``X``, leaving optionality to ``required``.

Absence from ``required`` already marks a field optional, so the published schema
does not spell nullability into the type as well. This keeps types single-valued
for editors and client generators, which read ``type`` as one string.
"""

def _plain(field_keys: dict, branch: dict) -> dict:
merged = {**branch, **field_keys, "type": branch["type"]}
if "default" in merged and merged["default"] is None:
del merged["default"]
return merged

return _rewrite_nullable(schema, _plain)


def _collapse_nullable(schema: dict) -> dict:
"""Rewrite ``X | None`` as a nullable ``type`` array, keeping null a legal value.

Airflow params always hold a value, so an unset optional field is validated as an
explicit null rather than as an absent key. This is the form Airflow's own params
documentation uses, and the trigger form dispatches on ``type`` alone.
"""

def _nullable(field_keys: dict, branch: dict) -> dict:
merged = {**branch, **field_keys, "type": [branch["type"], "null"]}
# null must be a permitted enum member too, or it fails validation on its own type.
enum = branch.get("enum")
if isinstance(enum, list) and None not in enum:
merged["enum"] = [*enum, None]
return merged

return _rewrite_nullable(schema, _nullable)


def _resolve_refs(schema: dict) -> dict:
"""Resolve all $ref/$defs in a JSON Schema, inlining definitions."""
defs = schema.get("$defs", {})
Expand Down Expand Up @@ -287,7 +357,7 @@ def get_schema(cls) -> dict:
Returns a flattened schema with all $ref/$defs resolved inline.
"""
raw = cls.get_config_type().model_json_schema()
return _resolve_refs(raw)
return _strip_nullable(_resolve_refs(raw))

@classmethod
def get_source_code(cls) -> str:
Expand Down Expand Up @@ -446,7 +516,7 @@ def get_config_type(cls) -> type[BaseModel]:
@classmethod
def get_schema(cls) -> dict:
raw = cls.get_config_type().model_json_schema()
return _resolve_refs(raw)
return _strip_nullable(_resolve_refs(raw))


class DefaultDagArgsConfig(BaseModel):
Expand Down
7 changes: 5 additions & 2 deletions tests/integration/project/dags/test_blueprints.py
Original file line number Diff line number Diff line change
Expand Up @@ -185,6 +185,7 @@ def render(self, config: LoadConfig) -> BashOperator:
class GreetConfig(BaseModel):
message: str = Field(default="hello", description="Greeting message")
repeat: int = Field(default=1, ge=1, description="Times to repeat")
suffix: str | None = Field(default=None, description="Optional text appended to the greeting")


class Greet(Blueprint[GreetConfig]):
Expand All @@ -202,15 +203,17 @@ def render(self, config: GreetConfig) -> TaskGroup:
task_id="template_greet",
bash_command=(
f"echo 'message={self.param('message')}' "
f"&& echo 'repeat={self.param('repeat')}'"
f"&& echo 'repeat={self.param('repeat')}' "
f"&& echo 'suffix={self.param('suffix')}'"
),
)

@task(task_id="resolved_greet")
def resolved_greet(**context):
cfg = self.resolve_config(config, context)
greeting = cfg.message if cfg.suffix is None else f"{cfg.message} {cfg.suffix}"
for i in range(cfg.repeat):
print(f"Resolved: {cfg.message} ({i + 1}/{cfg.repeat})")
print(f"Resolved: {greeting} ({i + 1}/{cfg.repeat})")

template_task >> resolved_greet()
return group
Expand Down
20 changes: 20 additions & 0 deletions tests/integration/test_cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@

from __future__ import annotations

import json
import os
import subprocess

Expand Down Expand Up @@ -151,3 +152,22 @@ def test_schema_transform(self):
def test_schema_nonexistent(self):
result = _run_blueprint("schema", "nonexistent", "--template-dir", DAGS_DIR)
assert result.returncode != 0

def test_optional_field_is_plain_type_and_not_required(self):
result = _run_blueprint("schema", "greet", "--template-dir", DAGS_DIR)
assert result.returncode == 0, f"blueprint schema failed:\n{result.stderr}"

schema = json.loads(result.stdout)
suffix = schema["properties"]["suffix"]
assert suffix["type"] == "string"
assert "default" not in suffix
assert "suffix" not in schema["required"]
assert "anyOf" not in result.stdout

def test_dag_args_optional_field_is_plain_type(self):
result = _run_blueprint("schema", "--dag-args", "--template-dir", DAGS_DIR)
assert result.returncode == 0, f"blueprint schema failed:\n{result.stderr}"

schema = json.loads(result.stdout)
assert schema["properties"]["schedule"]["type"] == "string"
assert "schedule" not in schema["required"]
34 changes: 34 additions & 0 deletions tests/integration/test_dag_execution.py
Original file line number Diff line number Diff line change
Expand Up @@ -242,3 +242,37 @@ def test_invalid_param_rejected(self, api_client: AirflowAPI):
f"Expected 400 for invalid param, got {resp.status_code}: {resp.text}"
)
assert "minimum" in resp.text.lower()

def test_explicit_null_optional_param_accepted(self, api_client: AirflowAPI):
"""`suffix` is unset in the YAML, so its param value is null and must stay valid."""
dag_id = "params_test"
_unpause_dag(api_client, dag_id)
run_id = _trigger_dag(api_client, dag_id, conf={"greet__suffix": None})
result = _wait_for_dag_run(api_client, dag_id, run_id)
assert result["state"] == "success", f"DAG run state: {result['state']}"

def test_overridden_optional_param_execution(self, api_client: AirflowAPI):
dag_id = "params_test"
_unpause_dag(api_client, dag_id)
run_id = _trigger_dag(api_client, dag_id, conf={"greet__suffix": "from-trigger"})
result = _wait_for_dag_run(api_client, dag_id, run_id)
assert result["state"] == "success", f"DAG run state: {result['state']}"

resp = api_client.get(f"/dags/{dag_id}/dagRuns/{run_id}/taskInstances")
assert resp.status_code == 200
task_instances = resp.json().get("task_instances", [])
failed = [ti["task_id"] for ti in task_instances if ti.get("state") != "success"]
assert not failed, f"Tasks did not succeed: {failed}"

def test_optional_param_rejects_wrong_type(self, api_client: AirflowAPI):
"""The nullable type array still constrains the non-null type."""
dag_id = "params_test"
_unpause_dag(api_client, dag_id)
logical_date = datetime.now(tz=timezone.utc).isoformat()
resp = api_client.post(
f"/dags/{dag_id}/dagRuns",
json={"logical_date": logical_date, "conf": {"greet__suffix": 123}},
)
assert resp.status_code == 400, (
f"Expected 400 for wrong-typed optional param, got {resp.status_code}: {resp.text}"
)
59 changes: 59 additions & 0 deletions tests/test_builder.py
Original file line number Diff line number Diff line change
Expand Up @@ -1446,6 +1446,65 @@ class MdConfig(BaseModel):
assert params["step__name"].schema.get("description_md") == "**Bold** description"


class TestConfigToParamsNullable:
class NullableConfig(BaseModel):
message: str
note: str | None = None
count: int | None = None
mode: Literal["fast", "slow"] | None = None
level: Literal[1, 2] | None = None

def test_optional_fields_are_nullable_type_arrays(self):
params = _config_to_params(self.NullableConfig, {"message": "hi"}, "step")
assert params["step__note"].schema["type"] == ["string", "null"]
assert params["step__count"].schema["type"] == ["integer", "null"]
assert "anyOf" not in params["step__note"].schema

def test_optional_literal_enum_admits_null(self):
params = _config_to_params(self.NullableConfig, {"message": "hi"}, "step")
assert params["step__mode"].schema["enum"] == ["fast", "slow", None]
assert params["step__level"].schema["enum"] == [1, 2, None]
assert params["step__level"].schema["type"] == ["integer", "null"]

def test_unset_optional_field_resolves(self):
params = _config_to_params(self.NullableConfig, {"message": "hi"}, "step")
for name in ("step__note", "step__count", "step__mode"):
assert params[name].resolve() is None

def test_optional_field_accepts_typed_override(self):
params = _config_to_params(self.NullableConfig, {"message": "hi"}, "step")
assert params["step__note"].resolve(value="hello") == "hello"
assert params["step__count"].resolve(value=7) == 7
assert params["step__mode"].resolve(value="fast") == "fast"

def test_optional_field_rejects_wrong_type(self):
from airflow.exceptions import ParamValidationError

params = _config_to_params(self.NullableConfig, {"message": "hi"}, "step")
with pytest.raises(ParamValidationError):
params["step__count"].resolve(value="not-a-number")

def test_required_field_still_rejects_null(self):
from airflow.exceptions import ParamValidationError

params = _config_to_params(self.NullableConfig, {"message": "hi"}, "step")
with pytest.raises(ParamValidationError):
params["step__message"].resolve(value=None)

def test_params_keep_null_legal_where_published_schema_drops_it(self):
"""A Param always holds a value, so null must stay valid there but not in the schema."""

class NullableBp(Blueprint[TestConfigToParamsNullable.NullableConfig]):
def render(self, config):
pass

published = NullableBp.get_schema()["properties"]["note"]
param = _config_to_params(self.NullableConfig, {"message": "hi"}, "step")["step__note"]

assert published["type"] == "string"
assert param.schema["type"] == ["string", "null"]


# --- Builder params integration tests ---


Expand Down
Loading