diff --git a/README.md b/README.md index a66af83..41ffd70 100644 --- a/README.md +++ b/README.md @@ -91,6 +91,19 @@ from blueprint import build_all_airflow_dags build_all_airflow_dags() ``` +#### On Airflow 3.4+: load YAML natively + +Airflow 3.4 can load Dags from other formats through Dag importers, so a loader file is no longer needed. Register `BlueprintDagImporter` as follows: + +```ini +[dag_processor] +dag_importer_configs = ["blueprint.importer.BlueprintDagImporter"] +``` + +Airflow then builds a Dag from every `*.dag.yaml` file in the bundle, finding blueprints and DAG args in the bundle's Python files, applying `.airflowignore`, and showing the YAML on the Code tab. Build errors show up as import errors for the YAML file itself. + +The importer accepts these `kwargs`: `extensions` (default `[".yaml"]`), `pattern` (default `*.dag.yaml`; widen it together with `extensions` to load `.yml` files), `profile`, `discover_entry_points`, and `embed_source`. An extension belongs to one importer, so do not register this alongside Airflow's built-in `YamlDagImporter` for the same files. + ### 4. Validate ```bash diff --git a/blueprint/importer.py b/blueprint/importer.py new file mode 100644 index 0000000..975204e --- /dev/null +++ b/blueprint/importer.py @@ -0,0 +1,240 @@ +"""Airflow Dag importer that builds Dags from Blueprint YAML files. + +Requires Airflow 3.4+, which introduced the ``AbstractDagImporter`` interface. Register it with: + +``` +[dag_processor] +dag_importer_configs = ["blueprint.importer.BlueprintDagImporter"] +``` +""" + +import logging +import re +import traceback +from collections.abc import Iterator +from fnmatch import fnmatch +from pathlib import Path +from typing import TYPE_CHECKING + +from airflow.sdk.importers import ( # ty: ignore[unresolved-import] + AbstractDagImporter, + DagDefinition, + DagImportError, + DagImportResult, + DagSourceCode, + FileDagDefinition, + find_file_dag_definitions, + get_file_suffix, +) + +from blueprint.builder import Builder, _load_dag_config, embed_source_yaml +from blueprint.errors import BlueprintError +from blueprint.registry import BlueprintRegistry + +if TYPE_CHECKING: + from airflow.dag_processing.bundles.base import BaseDagBundle + +logger = logging.getLogger(__name__) + +_DAG_MARKER_RE = re.compile(rb"^steps\s*:", re.MULTILINE) + + +class BlueprintDagImporter(AbstractDagImporter[FileDagDefinition]): + """Build Dags from Blueprint YAML files in a bundle. + + Blueprint templates and ``BlueprintDagArgs`` are discovered from the Python files in the + bundle (and from installed entry points), exactly as ``build_all_airflow_dags`` does. + + :param extensions: File extensions to handle. Defaults to ``.yaml``. Each extension is + routed to one importer, so do not register this alongside the built-in + ``YamlDagImporter`` for the same extension. To also handle ``.yml``, widen ``pattern`` + as well, because the default pattern only matches ``.yaml`` names. + :param pattern: Glob that file names must match to be listed. Defaults to ``*.dag.yaml``, + so ``blueprint.vars.yaml`` and other YAML in the bundle are left alone. Use ``*`` to + list every file with a supported extension. + :param profile: Active variable profile, as for ``build_all_airflow_dags``. + :param discover_entry_points: Whether to also discover blueprints from installed packages. + :param embed_source: Store each Dag's YAML in ``default_args`` for the Blueprint UI plugin. + """ + + supported_extensions = [".yaml"] # noqa: RUF012 + + def __init__( + self, + extensions: list[str] | None = None, + pattern: str = "*.dag.yaml", + profile: str | None = None, + discover_entry_points: bool = True, + embed_source: bool = True, + ) -> None: + """Configure the importer. + + Blueprint discovery is deferred to the first import, so creating the importer is cheap. + + :param extensions: File extensions to handle, with or without the leading dot. + :param pattern: Glob that file names must match to be listed. + :param profile: Active variable profile. + :param discover_entry_points: Whether to also discover blueprints from installed packages. + :param embed_source: Whether to store each Dag's YAML in ``default_args``. + """ + if extensions is not None: + self.supported_extensions = [ + e.lower() if e.startswith(".") else f".{e.lower()}" for e in extensions + ] + self.pattern = pattern + self.profile = profile + self.discover_entry_points = discover_entry_points + self.embed_source = embed_source + self._registries: dict[Path, BlueprintRegistry] = {} + + def can_handle(self, definition: DagDefinition | str | Path) -> bool: + """Check whether this importer handles a definition, path, or file name. + + :param definition: A Dag definition, a path, or a file name. + :returns: True if its suffix is one of :attr:`supported_extensions`. + """ + suffix = get_file_suffix(definition) + return suffix in self.supported_extensions if suffix else False + + def list_dag_definitions( + self, + bundle: "BaseDagBundle", + *, + safe_mode: bool = True, + ) -> Iterator[FileDagDefinition | DagImportError]: + """List the Blueprint YAML files in a bundle. + + Files are found with Airflow's own walker, so ``.airflowignore`` is honored. Only files + whose name matches :attr:`pattern` and that pass :meth:`might_contain_dag` are yielded. + + :param bundle: The bundle to list. Only ``bundle.path`` is used, and it may be a + directory or a single file. + :param safe_mode: Whether to skip files that clearly hold no Dag. + :returns: An iterator of file definitions to import. + """ + for definition in find_file_dag_definitions(Path(bundle.path), self.supported_extensions): + if fnmatch(Path(repr(definition)).name, self.pattern) and self.might_contain_dag( + definition, safe_mode + ): + yield definition + + def might_contain_dag(self, definition: DagDefinition, safe_mode: bool) -> bool: + """Skip YAML files with no top-level ``steps`` key when safe mode is on. + + :param definition: The definition to check. + :param safe_mode: Whether safe mode is on. When off, every definition is kept. + :returns: False only if safe mode is on and the file has no ``steps:`` key. A file that + cannot be read is kept so that :meth:`import_definition` reports the error. + """ + if not safe_mode: + return True + try: + return _DAG_MARKER_RE.search(definition.read_bytes()) is not None + except OSError: + return True + + def import_definition( + self, + definition: FileDagDefinition, + bundle: "BaseDagBundle", + ) -> DagImportResult: + """Build the Dag described by one YAML file. + + Failures are reported as :class:`~airflow.sdk.importers.DagImportError` entries in the + result rather than raised, so Airflow shows them as import errors. A file with no + ``steps`` field is not a Dag definition and yields an empty, successful result. + + :param definition: The YAML file to import. + :param bundle: The bundle holding the file, which is the root that blueprints, DAG args + and variables are searched in. + :returns: A result holding the built Dag, or the errors that prevented it. + """ + result = DagImportResult(definition=definition) + ref = repr(definition) + yaml_path = Path(ref) + bundle_root = self._bundle_root(bundle) + + try: + with definition.import_context(): + dag_config = _load_dag_config( + yaml_path, + bundle_root, + render_templates=True, + template_context=None, + profile=self.profile, + ) + if dag_config is None: + return result + dag = Builder(bp_registry=self._registry(bundle_root)).build( + dag_config, source_path=yaml_path + ) + except BlueprintError as exc: + result.errors.append( + DagImportError( + source_reference=ref, + message=str(exc), + error_type="import", + stacktrace=traceback.format_exc(), + ) + ) + return result + except Exception as exc: + result.errors.append( + DagImportError( + source_reference=ref, + message=f"{type(exc).__name__}: {exc}", + error_type="import", + stacktrace=traceback.format_exc(), + ) + ) + return result + + if self.embed_source: + embed_source_yaml(dag, yaml_path) + result.dags.append(dag) + logger.debug("Built DAG '%s' from %s", dag.dag_id, ref) + return result + + def get_source_code( + self, + definition: DagDefinition, + dag_id: str | None = None, # noqa: ARG002 + ) -> DagSourceCode: + """Return the YAML source shown on the Dag's Code tab. + + :param definition: The YAML file. + :param dag_id: Ignored, because a file defines exactly one Dag. + :returns: The file's text, with language ``yaml``. + """ + return DagSourceCode(source_code=definition.read_text(encoding="utf-8"), language="yaml") + + @staticmethod + def _bundle_root(bundle: "BaseDagBundle") -> Path: + """Return the directory blueprints, DAG args and variables are searched in. + + Parse processes and workers may scope ``bundle.path`` to the single file being parsed, + in which case the file's parent directory is used. + + :param bundle: The bundle being imported from. + :returns: The bundle directory. + """ + bundle_path = Path(bundle.path) + return bundle_path.parent if bundle_path.is_file() else bundle_path + + def _registry(self, root: Path) -> BlueprintRegistry: + """Return the blueprint registry for a directory, discovering it on first use. + + Registries are cached per importer instance, so discovery runs once per directory in + each process. + + :param root: The directory to discover blueprints in. + :returns: A registry with discovery already run. + """ + if root not in self._registries: + reg = BlueprintRegistry( + template_dirs=[root], + discover_entry_points=self.discover_entry_points, + ) + reg.discover(force=True) + self._registries[root] = reg + return self._registries[root] diff --git a/tests/test_importer.py b/tests/test_importer.py new file mode 100644 index 0000000..b808000 --- /dev/null +++ b/tests/test_importer.py @@ -0,0 +1,178 @@ +"""Tests for the native Airflow Dag importer.""" + +from types import SimpleNamespace + +import pytest + +pytest.importorskip("airflow.sdk.importers", reason="DagImporter needs Airflow 3.4+") + +from airflow.sdk.importers import DagImporterRegistry, FilesystemDagDefinition + +from blueprint.builder import SOURCE_YAML_KEY +from blueprint.importer import BlueprintDagImporter +from tests.conftest import write_dag_args, write_dag_yaml, write_stub_blueprint + + +def _bundle(path): + return SimpleNamespace(name="test", path=path) + + +@pytest.fixture +def bundle_dir(tmp_path): + write_stub_blueprint(tmp_path) + return tmp_path + + +def _import(importer, bundle_dir, name): + definition = FilesystemDagDefinition(path=bundle_dir / name) + return importer.import_definition(definition, _bundle(bundle_dir)) + + +class TestListing: + def test_lists_only_dag_yaml_files(self, bundle_dir): + """Only files matching the default ``*.dag.yaml`` pattern are listed; other YAML in the bundle is ignored.""" + write_dag_yaml(bundle_dir, "one") + (bundle_dir / "blueprint.vars.yaml").write_text("steps: {}\n") + (bundle_dir / "notes.yaml").write_text("a: 1\n") + + found = list(BlueprintDagImporter().list_dag_definitions(_bundle(bundle_dir))) + + assert [d.path.name for d in found] == ["one.dag.yaml"] + + def test_safe_mode_skips_files_without_steps(self, bundle_dir): + """With safe mode on, a ``.dag.yaml`` with no ``steps`` key is skipped; with it off, the file is still listed.""" + (bundle_dir / "other.dag.yaml").write_text("dag_id: x\n") + + importer = BlueprintDagImporter() + + assert list(importer.list_dag_definitions(_bundle(bundle_dir), safe_mode=True)) == [] + assert len(list(importer.list_dag_definitions(_bundle(bundle_dir), safe_mode=False))) == 1 + + def test_respects_airflowignore(self, bundle_dir): + """Files under a path matched by ``.airflowignore`` are not listed.""" + write_dag_yaml(bundle_dir, "kept") + write_dag_yaml(bundle_dir / "skipped", "ignored") + (bundle_dir / ".airflowignore").write_text("skipped\n") + + found = list(BlueprintDagImporter().list_dag_definitions(_bundle(bundle_dir))) + + assert [d.path.name for d in found] == ["kept.dag.yaml"] + + def test_custom_pattern(self, bundle_dir): + """A custom ``pattern`` replaces the default, so a plain ``.yaml`` file is listed when the pattern is ``*``.""" + (bundle_dir / "plain.yaml").write_text("dag_id: p\nsteps:\n s1:\n blueprint: stub\n") + + found = list(BlueprintDagImporter(pattern="*").list_dag_definitions(_bundle(bundle_dir))) + + assert [d.path.name for d in found] == ["plain.yaml"] + + def test_can_handle(self): + """By default only ``.yaml`` names are claimed, not ``.yml`` or Python files.""" + importer = BlueprintDagImporter() + assert importer.can_handle("a.dag.yaml") + assert not importer.can_handle("a.yml") + assert not importer.can_handle("a.py") + + def test_extensions_override(self): + """The ``extensions`` argument replaces the default, claiming ``.yml`` and dropping ``.yaml``.""" + importer = BlueprintDagImporter(extensions=["yml"]) + assert importer.can_handle("a.dag.yml") + assert not importer.can_handle("a.dag.yaml") + + +class TestImport: + def test_builds_dag(self, bundle_dir): + """A valid ``.dag.yaml`` imports into a single Dag with the right id and tasks and no errors.""" + write_dag_yaml(bundle_dir, "my_dag") + + result = _import(BlueprintDagImporter(), bundle_dir, "my_dag.dag.yaml") + + assert result.success + assert [d.dag_id for d in result.dags] == ["my_dag"] + assert set(result.dags[0].task_dict) == {"s1"} + + def test_embeds_source_by_default(self, bundle_dir): + """The YAML text is stored in ``default_args`` for the UI plugin, and ``embed_source=False`` turns that off.""" + path = write_dag_yaml(bundle_dir, "my_dag") + + dag = _import(BlueprintDagImporter(), bundle_dir, "my_dag.dag.yaml").dags[0] + assert dag.default_args[SOURCE_YAML_KEY] == path.read_text() + + dag = _import(BlueprintDagImporter(embed_source=False), bundle_dir, "my_dag.dag.yaml").dags[ + 0 + ] + assert SOURCE_YAML_KEY not in dag.default_args + + def test_uses_dag_args_from_bundle(self, bundle_dir): + """A ``BlueprintDagArgs`` template defined in the bundle is found and applied to the built Dag.""" + write_dag_args(bundle_dir, "Team", default=True) + write_dag_yaml(bundle_dir, "my_dag") + + dag = _import(BlueprintDagImporter(), bundle_dir, "my_dag.dag.yaml").dags[0] + + assert "Team" in dag.tags + + def test_unknown_blueprint_is_import_error(self, bundle_dir): + """Referencing a blueprint that does not exist is reported as an import error naming it, and no Dag is built.""" + (bundle_dir / "bad.dag.yaml").write_text( + "dag_id: bad\nsteps:\n s1:\n blueprint: nope\n" + ) + + result = _import(BlueprintDagImporter(), bundle_dir, "bad.dag.yaml") + + assert not result.success + assert result.dags == [] + assert "nope" in result.errors[0].message + + def test_invalid_yaml_is_import_error(self, bundle_dir): + """A file that is not valid YAML is reported as an import error instead of raising.""" + (bundle_dir / "bad.dag.yaml").write_text("dag_id: x\n bad: indent\n") + + result = _import(BlueprintDagImporter(), bundle_dir, "bad.dag.yaml") + + assert not result.success + + def test_file_without_steps_yields_nothing(self, bundle_dir): + """A YAML file with no ``steps`` is not a Dag definition and imports as an empty, successful result.""" + (bundle_dir / "other.dag.yaml").write_text("dag_id: x\n") + + result = _import(BlueprintDagImporter(), bundle_dir, "other.dag.yaml") + + assert result.success + assert result.dags == [] + + def test_file_scoped_bundle_path(self, bundle_dir): + """When the bundle path is the single file being parsed, the file's directory is used to find blueprints.""" + path = write_dag_yaml(bundle_dir, "my_dag") + + result = BlueprintDagImporter().import_definition( + FilesystemDagDefinition(path=path), _bundle(path) + ) + + assert [d.dag_id for d in result.dags] == ["my_dag"] + + def test_source_code(self, bundle_dir): + """``get_source_code`` returns the file's text with language ``yaml``, as shown on the Code tab.""" + path = write_dag_yaml(bundle_dir, "my_dag") + + source = BlueprintDagImporter().get_source_code(FilesystemDagDefinition(path=path)) + + assert source.language == "yaml" + assert source.source_code == path.read_text() + + +class TestRegistry: + def test_configured_via_dag_importer_configs(self, bundle_dir): + """Registering the importer by classpath alone routes ``.yaml`` files to it and lists them from a bundle.""" + registry = DagImporterRegistry(register_defaults=True) + registry.register_specs( + ["blueprint.importer.BlueprintDagImporter"], + context="test", + ) + write_dag_yaml(bundle_dir, "my_dag") + + importer = registry.get_importer(bundle_dir / "my_dag.dag.yaml") + + assert isinstance(importer, BlueprintDagImporter) + items = [d for _, d in registry.list_dag_definitions(_bundle(bundle_dir))] + assert [d.path.name for d in items] == ["my_dag.dag.yaml"]