Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
27 changes: 26 additions & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -116,7 +116,7 @@ See the [examples README](examples/README.md) for full setup details.

By default, DAG YAML files support `schedule` and `description` at the top level (alongside `dag_id` and `steps`). For more control over DAG construction, define a `BlueprintDagArgs` template.

A `BlueprintDagArgs` subclass works like a Blueprint but for DAG-level arguments. It defines a Pydantic config model whose fields become the valid top-level YAML fields, and a `render()` method that returns DAG constructor kwargs. At most one may exist per project.
A `BlueprintDagArgs` subclass works like a Blueprint but for DAG-level arguments. It defines a Pydantic config model whose fields become the valid top-level YAML fields, and a `render()` method that returns DAG constructor kwargs.

```python
from datetime import timedelta
Expand Down Expand Up @@ -158,6 +158,28 @@ steps:

When no `BlueprintDagArgs` is defined, the built-in `DefaultDagArgs` provides `schedule` and `description` pass-through.

A project can define several templates. A DAG uses the one defined closest above it, so a subdirectory overrides its parents. The directory that scopes a template is the one holding the `.py` file that defines it:

```
dags/
dag_args.py ProjectDagArgs
customer.dag.yaml -> ProjectDagArgs
sandbox/
dag_args.py SandboxDagArgs
probe.dag.yaml -> SandboxDagArgs
```

`blueprint lint` prints which template each DAG resolved to.

Two templates in one directory are ambiguous and raise an error, as do two sharing a name — set `name = "..."` on the class to register one under a different name. A DAG with no template above it uses the one declared as the fallback, or the only template defined:

```python
class ProjectDagArgs(BlueprintDagArgs[ProjectDagArgsConfig], default=True):
...
```

DAGs built directly with the `Builder` API resolve from the Python file that builds them: pass `source_path=__file__` to use the template above it.

## Template Versioning

### How names and versions are determined
Expand Down Expand Up @@ -853,6 +875,9 @@ blueprint vars pipeline.dag.yaml --profile prod --unused
# step template, or "dag_args" for DAG-level fields via `blueprint schema --dag-args`)
blueprint schema extract > extract.schema.json

# A project with several DAG args templates has one DAG schema per template
blueprint schema --dag-args sandbox_dag_args > sandbox.dag.schema.json

# Create new DAG interactively
blueprint new
```
Expand Down
6 changes: 6 additions & 0 deletions blueprint/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,14 +19,17 @@
ConfigurationError,
CyclicDependencyError,
CyclicVariableError,
DagArgsNotFoundError,
DuplicateBlueprintError,
DuplicateDagArgsError,
DuplicateDAGIdError,
IncompleteVariableError,
InvalidDependencyError,
InvalidVariableNameError,
InvalidVariableValueError,
InvalidVersionError,
MultipleDagArgsError,
MultipleDefaultDagArgsError,
NonContiguousVersionError,
ProfileError,
UndefinedVariableError,
Expand Down Expand Up @@ -72,10 +75,12 @@
"CyclicDependencyError",
"CyclicVariableError",
"DAGConfig",
"DagArgsNotFoundError",
"DefaultDagArgs",
"Dict",
"DuplicateBlueprintError",
"DuplicateDAGIdError",
"DuplicateDagArgsError",
"Field",
"IncompleteVariableError",
"InvalidDependencyError",
Expand All @@ -84,6 +89,7 @@
"InvalidVersionError",
"List",
"MultipleDagArgsError",
"MultipleDefaultDagArgsError",
"NonContiguousVersionError",
"OnDagBuilt",
"Optional",
Expand Down
90 changes: 79 additions & 11 deletions blueprint/builder.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@
import yaml
from pydantic import BaseModel, ConfigDict, Field, ValidationError, field_validator, model_validator

from blueprint.core import TaskOrGroup
from blueprint.core import BlueprintDagArgs, TaskOrGroup
from blueprint.errors import (
BlueprintError,
ConfigurationError,
Expand All @@ -21,6 +21,7 @@
InvalidDependencyError,
)
from blueprint.registry import BlueprintRegistry, registry
from blueprint.utils import display_path

if TYPE_CHECKING:
from airflow import DAG
Expand Down Expand Up @@ -187,11 +188,15 @@ def _ensure_start_date(dag_kwargs: dict[str, Any]) -> None:
parsed = parsed.replace(tzinfo=timezone.utc)
dag_kwargs["start_date"] = parsed

def build(self, config: DAGConfig) -> "DAG":
def build(self, config: DAGConfig, source_path: str | Path | None = None) -> "DAG":
"""Build a DAG from a DAGConfig.

Args:
config: The parsed and validated DAG configuration
source_path: Path of the file this DAG is defined in, whose parent
directories are searched for the DAG args template. Without it,
only the project fallbacks apply, so a Python file building DAGs
directly should pass ``source_path=__file__``.

Returns:
A fully wired Airflow DAG
Expand All @@ -200,10 +205,9 @@ def build(self, config: DAGConfig) -> "DAG":

self.validate_dependencies(config)

dag_args_cls = self._registry.get_dag_args()
config_type = dag_args_cls.get_config_type()
extra = config.get_extra_fields()
validated_dag_args = config_type(**extra)
dag_args_cls = self._registry.resolve_dag_args(source_path)
validated_dag_args = self._validate_against(dag_args_cls, config, source_path)
dag_args_yaml = self._dag_args_summary(dag_args_cls)

instance = dag_args_cls()
dag_kwargs = instance.render(validated_dag_args)
Expand Down Expand Up @@ -236,7 +240,7 @@ def build(self, config: DAGConfig) -> "DAG":

with dag:
for step_name, step_config in config.steps.items():
rendered[step_name] = self._render_step(step_name, step_config)
rendered[step_name] = self._render_step(step_name, step_config, dag_args_yaml)

for step_name, step_config in config.steps.items():
for dep_name in step_config.depends_on:
Expand All @@ -246,6 +250,62 @@ def build(self, config: DAGConfig) -> "DAG":

return dag

def _dag_args_summary(self, dag_args_cls: type[BlueprintDagArgs]) -> str:
"""Describe which DAG args template built a DAG, for the Airflow task view."""
name = dag_args_cls.template_name()
summary = {"template": name}

location = self._registry.dag_args_location(name)
if location:
summary["defined_in"] = display_path(location)

return yaml.dump(summary, default_flow_style=False, sort_keys=False)

def validate_dag_args(
self, config: DAGConfig, source_path: str | Path | None = None
) -> BaseModel:
"""Validate a DAG's top-level fields against the template that applies to it.

Args:
config: The parsed DAG configuration.
source_path: Path of the file this DAG is defined in, whose parent
directories are searched for the DAG args template.

Returns:
The validated DAG args config instance.

Raises:
ConfigurationError: If the top-level fields do not match the template.
"""
dag_args_cls = self._registry.resolve_dag_args(source_path)
return self._validate_against(dag_args_cls, config, source_path)

def _validate_against(
self,
dag_args_cls: type[BlueprintDagArgs],
config: DAGConfig,
source_path: str | Path | None,
) -> BaseModel:
"""Validate a DAG's top-level fields against an already-resolved template."""
try:
return dag_args_cls.get_config_type()(**config.get_extra_fields())
except ValidationError as e:
name = dag_args_cls.template_name()
location = self._registry.dag_args_location(name)
where = f" ({display_path(location)})" if location else ""

lines = [f"DAG arguments rejected by template '{name}'{where}:"]
for err in e.errors():
field = ".".join(str(loc) for loc in err["loc"])
lines.append(f" - '{field}': {err['msg']}" if field else f" - {err['msg']}")

accepted = ", ".join(sorted(dag_args_cls.get_config_type().model_fields))
raise ConfigurationError(
"\n".join(lines),
file_path=Path(source_path) if source_path else None,
suggestions=[f"Accepted DAG arguments: {accepted or 'none'}"],
) from e

def build_from_yaml(
self,
path: str | Path,
Expand Down Expand Up @@ -287,7 +347,7 @@ def build_from_yaml(
)

dag_config = DAGConfig.model_validate(raw_config)
dag = self.build(dag_config)
dag = self.build(dag_config, source_path=yaml_path)

if self._on_dag_built:
self._on_dag_built(dag, yaml_path)
Expand Down Expand Up @@ -372,6 +432,7 @@ def _render_step(
self,
step_name: str,
step_config: StepConfig,
dag_args_yaml: str,
) -> TaskOrGroup:
"""Render a single step by instantiating its blueprint."""
from pydantic import ValidationError
Expand Down Expand Up @@ -436,7 +497,7 @@ def _render_step(

source_code = bp_class.get_source_code()

self._inject_step_context(result, step_yaml, source_code)
self._inject_step_context(result, step_yaml, source_code, dag_args_yaml)

return result

Expand All @@ -445,6 +506,7 @@ def _inject_step_context(
rendered: TaskOrGroup,
step_yaml: str,
source_code: str,
dag_args_yaml: str,
) -> None:
"""Inject step config and blueprint source into all tasks for Airflow UI visibility."""
from airflow.models import BaseOperator
Expand All @@ -470,10 +532,15 @@ def _collect_operators(node: TaskOrGroup) -> list[BaseOperator]:
for task in tasks:
task.blueprint_step_config = step_yaml # type: ignore[attr-defined]
task.blueprint_step_code = source_code # type: ignore[attr-defined]
task.blueprint_dag_args = dag_args_yaml # type: ignore[attr-defined]

existing_fields = getattr(task, "template_fields", ()) or ()
new_fields = []
for field_name in ("blueprint_step_config", "blueprint_step_code"):
for field_name in (
"blueprint_step_config",
"blueprint_step_code",
"blueprint_dag_args",
):
if field_name not in existing_fields:
new_fields.append(field_name)
if new_fields:
Expand All @@ -483,6 +550,7 @@ def _collect_operators(node: TaskOrGroup) -> list[BaseOperator]:
**getattr(task, "template_fields_renderers", {}),
"blueprint_step_config": "yaml",
"blueprint_step_code": "py",
"blueprint_dag_args": "yaml",
}

def _apply_trigger_rule(self, rendered: TaskOrGroup, trigger_rule: str) -> None:
Expand Down Expand Up @@ -621,7 +689,7 @@ def build_all_airflow_dags(

dag_config = DAGConfig.model_validate(raw_config)
_check_duplicate_dag_id(dag_config.dag_id, yaml_path, dag_id_to_file)
dag = builder.build(dag_config)
dag = builder.build(dag_config, source_path=yaml_path)

if on_dag_built:
on_dag_built(dag, yaml_path)
Expand Down
Loading