Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 7 additions & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -334,7 +334,9 @@ my_project = "my_project.blueprints"

Note that this applies to the discoverability of Blueprint templates. A template can still import from another module that's not included in the `entry-point`.

## Airflow Rendered Templates
## Airflow UI Integration

### Rendered Templates tab

Every task instance gets two extra fields visible in Airflow's "Rendered Template" tab:

Expand All @@ -343,6 +345,10 @@ Every task instance gets two extra fields visible in Airflow's "Rendered Templat

This makes it easy to understand what generated each task instance without leaving the Airflow UI.

### YAML tab

Airflow's Code tab shows the loader file, not the YAML a DAG was built from. On Airflow 3.1+, installing `airflow-blueprint` adds a **YAML** tab to each DAG page that shows the source YAML. No configuration is needed; the plugin registers itself. Pass `embed_source=False` to `build_all_airflow_dags()` to turn it off.

## Runtime Parameter Overrides

Blueprints that set `supports_params = True` have their config fields registered as Airflow [DAG params](https://airflow.apache.org/docs/apache-airflow/stable/core-concepts/params.html), namespaced as `{step}__{field}`. When you trigger a DAG from the Airflow UI, the trigger form shows those fields pre-filled with YAML defaults — users can override any value before running.
Expand Down
79 changes: 60 additions & 19 deletions blueprint/builder.py
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,8 @@

DEFAULT_START_DATE = datetime(2024, 1, 1, tzinfo=timezone.utc)

SOURCE_YAML_KEY = "blueprint_source"

_PARAM_SCHEMA_KEYS = frozenset(
{
"type",
Expand Down Expand Up @@ -576,6 +578,53 @@ def _check_duplicate_dag_id(dag_id: str, yaml_path: Path, dag_id_to_file: dict[s
raise DuplicateDAGIdError(dag_id, [dag_id_to_file[dag_id], yaml_path])


def _load_dag_config(
yaml_path: Path,
search_root: Path,
render_templates: bool,
template_context: dict[str, Any] | None,
profile: str | None,
) -> DAGConfig | None:
"""Render, resolve variables in, and validate one DAG YAML.

Returns None for YAML files that are not DAG definitions (no ``steps`` field).
"""
from blueprint import vars as bp_vars
from blueprint.loaders import render_yaml_template

if render_templates:
raw_config, _rendered = render_yaml_template(
yaml_path,
context={"profile": profile, **(template_context or {})},
use_airflow_context=True,
)
else:
raw_config = yaml.safe_load(yaml_path.read_text())

if not raw_config or not isinstance(raw_config, dict) or "steps" not in raw_config:
logger.debug("Skipping %s: no 'steps' field", yaml_path.name)
return None

raw_config, _resolved = bp_vars.resolve(
raw_config, yaml_path, profile=profile, search_root=search_root
)
return DAGConfig.model_validate(raw_config)


def embed_source_yaml(dag: "DAG", yaml_path: Path) -> None:
"""Store the DAG's YAML text in ``default_args`` under ``blueprint_source``.

``default_args`` survives DAG serialization, so the Airflow UI plugin can show
the YAML a DAG version was built from without reading the dags folder.
Operators ignore ``default_args`` keys they do not accept, so no task sees it.

Args:
dag: The built DAG.
yaml_path: The DAG's YAML file.
"""
dag.default_args = {**dag.default_args, SOURCE_YAML_KEY: yaml_path.read_text()}


def build_all_airflow_dags(
search_path: str | Path | None = None,
register_globals: dict | None = None,
Expand All @@ -587,6 +636,7 @@ def build_all_airflow_dags(
skip_invalid_dags: bool = False,
discover_entry_points: bool = True,
profile: str | None = None,
embed_source: bool = True,
) -> list["DAG"]:
"""Discover and build all DAGs from YAML files.

Expand Down Expand Up @@ -622,6 +672,8 @@ def build_all_airflow_dags(
group. Ignored when ``bp_registry`` is supplied directly.
profile: Active variable profile. Only needed when a referenced variable
declares a per-profile value.
embed_source: Store each DAG's YAML text in ``default_args`` under
``blueprint_source`` so the Airflow UI plugin can show it.

Returns:
List of built DAGs
Expand All @@ -634,8 +686,7 @@ def build_all_airflow_dags(
build_all_airflow_dags()
```
"""
from blueprint import vars as bp_vars
from blueprint.loaders import discover_yaml_files, render_yaml_template
from blueprint.loaders import discover_yaml_files

if register_globals is None:
frame = inspect.currentframe()
Expand Down Expand Up @@ -669,28 +720,18 @@ def build_all_airflow_dags(
skipped_files: set[str] = set()
for yaml_path in yaml_files:
try:
if render_templates:
raw_config, _rendered = render_yaml_template(
yaml_path,
context={"profile": profile, **(template_context or {})},
use_airflow_context=True,
)
else:
raw_content = yaml_path.read_text()
raw_config = yaml.safe_load(raw_content)

if not raw_config or not isinstance(raw_config, dict) or "steps" not in raw_config:
logger.debug("Skipping %s: no 'steps' field", yaml_path.name)
continue

raw_config, _resolved = bp_vars.resolve(
raw_config, yaml_path, profile=profile, search_root=resolved_path
dag_config = _load_dag_config(
yaml_path, resolved_path, render_templates, template_context, profile
)
if dag_config is None:
continue

dag_config = DAGConfig.model_validate(raw_config)
_check_duplicate_dag_id(dag_config.dag_id, yaml_path, dag_id_to_file)
dag = builder.build(dag_config, source_path=yaml_path)

if embed_source:
embed_source_yaml(dag, yaml_path)

if on_dag_built:
on_dag_built(dag, yaml_path)

Expand Down
115 changes: 115 additions & 0 deletions blueprint/plugin.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,115 @@
"""Airflow UI plugin: a YAML tab on the DAG page showing the source a DAG was built from.

Registered through the ``airflow.plugins`` entry point. The tab needs
``external_views``, which Airflow added in 3.1; older versions ignore the
attribute. Airflow 2 has no FastAPI, so the app is skipped there.
"""

import html
from typing import TYPE_CHECKING, Any

from airflow.plugins_manager import AirflowPlugin

from blueprint.builder import SOURCE_YAML_KEY

if TYPE_CHECKING:
from fastapi import FastAPI

URL_PREFIX = "/blueprint"

NOT_FOUND_MESSAGE = "This DAG was not built from a Blueprint YAML file."

PAGE = """<!doctype html>
<html>
<head>
<meta charset="utf-8">
<title>{title}</title>
<style>
body {{ margin: 0; font-family: ui-monospace, SFMono-Regular, Menlo, monospace; }}
pre {{ margin: 0; padding: 16px; font-size: 13px; line-height: 1.5; overflow: auto; }}
</style>
</head>
<body><pre>{body}</pre></body>
</html>
"""


def source_yaml_from_default_args(default_args: Any) -> str | None:
"""Pull the embedded YAML out of serialized ``default_args``.

Serialized dicts are wrapped as ``{"__type": "dict", "__var": {...}}``; a plain
dict is accepted too.

Args:
default_args: The ``default_args`` entry of a serialized DAG.

Returns:
The YAML text, or None when the DAG has none.
"""
if isinstance(default_args, dict) and "__var" in default_args:
default_args = default_args["__var"]
if not isinstance(default_args, dict):
return None
source = default_args.get(SOURCE_YAML_KEY)
return source if isinstance(source, str) else None


def source_yaml(dag_id: str) -> str | None:
"""Read the YAML ``build_all_airflow_dags`` embedded in the latest serialized DAG.

Args:
dag_id: DAG id shown in the Airflow UI.

Returns:
The YAML text, or None when the DAG is unknown or has none.
"""
from airflow.models.serialized_dag import SerializedDagModel
from airflow.utils.session import create_session

with create_session() as session:
serialized = SerializedDagModel.get(dag_id, session=session)
if serialized is None or serialized.data is None:
return None
return source_yaml_from_default_args(serialized.data["dag"].get("default_args"))


def create_app() -> "FastAPI":
"""Build the FastAPI app that serves DAG YAML pages.

Returns:
The configured FastAPI app.
"""
from fastapi import FastAPI
from fastapi.responses import HTMLResponse

app = FastAPI(title="Blueprint")

@app.get("/dags/{dag_id}/yaml", response_class=HTMLResponse)
def dag_yaml(dag_id: str) -> str:
text = source_yaml(dag_id) or NOT_FOUND_MESSAGE
return PAGE.format(title=html.escape(dag_id), body=html.escape(text))

return app


try:
_fastapi_apps = [{"app": create_app(), "url_prefix": URL_PREFIX, "name": "Blueprint"}]
_external_views = [
{
"name": "YAML",
"href": f"{URL_PREFIX}/dags/{{DAG_ID}}/yaml",
"destination": "dag",
"url_route": "blueprint",
}
]
except ImportError:
_fastapi_apps = []
_external_views = []


class BlueprintPlugin(AirflowPlugin):
"""Show the YAML behind each Blueprint DAG in the Airflow UI."""

name = "blueprint"
fastapi_apps = _fastapi_apps
external_views = _external_views
Binary file added docs/images/plugin-blueprint-tab.png
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
3 changes: 3 additions & 0 deletions pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,9 @@ classifiers = [
[project.scripts]
blueprint = "blueprint.cli:cli"

[project.entry-points."airflow.plugins"]
blueprint = "blueprint.plugin:BlueprintPlugin"

[project.urls]
"Homepage" = "https://github.com/astronomer/blueprint"
"Repository" = "https://github.com/astronomer/blueprint"
Expand Down
39 changes: 39 additions & 0 deletions tests/integration/test_plugin.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
"""The Blueprint UI plugin against a running Airflow: tab, embedded YAML, and page."""

import httpx
import pytest

pytestmark = pytest.mark.integration

SIMPLE_YAML = (
"dag_id: simple_pipeline\nteam: data-eng\nsteps:\n process:\n blueprint: transform\n"
)


def test_plugin_registers_a_yaml_tab(api_client):
resp = api_client.get("/plugins")
assert resp.status_code == 200, resp.text
(plugin,) = [p for p in resp.json()["plugins"] if p["name"] == "blueprint"]
(view,) = plugin["external_views"]
assert view["name"] == "YAML"
assert view["destination"] == "dag"
assert view["href"] == "/blueprint/dags/{DAG_ID}/yaml"


def test_dag_carries_no_extra_tags(api_client):
resp = api_client.get("/dags/simple_pipeline")
assert resp.status_code == 200, resp.text
assert not {t for t in api_client.get_tags(resp.json()) if t.startswith("blueprint")}


def test_yaml_page_serves_the_embedded_source(api_client):
resp = httpx.get(f"{api_client.base_url}/blueprint/dags/simple_pipeline/yaml", timeout=30)
assert resp.status_code == 200, resp.text
assert "<title>simple_pipeline</title>" in resp.text
assert SIMPLE_YAML in resp.text


def test_yaml_page_explains_a_dag_without_yaml(api_client):
resp = httpx.get(f"{api_client.base_url}/blueprint/dags/nope/yaml", timeout=30)
assert resp.status_code == 200
assert "not built from a Blueprint YAML file" in resp.text
36 changes: 33 additions & 3 deletions tests/test_builder.py
Original file line number Diff line number Diff line change
Expand Up @@ -1026,9 +1026,9 @@ def test_build_all_dag_args_from_nearest_directory(self, tmp_path):
render_templates=False,
)
tags_by_dag = {dag.dag_id: set(dag.tags) for dag in dags}
assert tags_by_dag["missions_dag"] == {"MissionDagArgs"}
assert tags_by_dag["observatory_dag"] == {"ObservatoryDagArgs"}
assert tags_by_dag["orphan_dag"] == {"MissionDagArgs"}
assert "MissionDagArgs" in tags_by_dag["missions_dag"]
assert "ObservatoryDagArgs" in tags_by_dag["observatory_dag"]
assert "MissionDagArgs" in tags_by_dag["orphan_dag"]

def test_build_all_ambiguous_dag_args_raises(self, tmp_path):
from blueprint.builder import build_all_airflow_dags
Expand Down Expand Up @@ -1763,3 +1763,33 @@ def test_resolve_validates(self):
context = {"params": {"greet__count": -1}}
with pytest.raises(ValidationError):
bp.resolve_config(config, context)


class TestEmbeddedSourceYaml:
"""DAGs built from YAML carry that YAML in default_args for the UI plugin."""

def _build(self, tmp_path, **kwargs):
from blueprint.builder import build_all_airflow_dags

write_stub_blueprint(tmp_path)
self.yaml_path = write_dag_yaml(tmp_path / "team", "meta_test")
return build_all_airflow_dags(
search_path=tmp_path, register_globals={}, render_templates=False, **kwargs
)

def test_default_args_carry_the_raw_yaml(self, tmp_path):
(dag,) = self._build(tmp_path)
assert dag.default_args["blueprint_source"] == self.yaml_path.read_text()

def test_opt_out(self, tmp_path):
(dag,) = self._build(tmp_path, embed_source=False)
assert "blueprint_source" not in dag.default_args

def test_yaml_survives_serialization_and_tasks_ignore_it(self, tmp_path):
from airflow.serialization.serialized_objects import SerializedDAG

(dag,) = self._build(tmp_path)
restored = SerializedDAG.from_dict(SerializedDAG.to_dict(dag))
assert restored.default_args["blueprint_source"] == self.yaml_path.read_text()
for task in restored.tasks:
assert not hasattr(task, "blueprint_source")
Loading
Loading