From bd9be97f70e84d3d7d46a037d248ce9d84c82f80 Mon Sep 17 00:00:00 2001 From: LIU ZHE YOU Date: Wed, 30 Sep 2026 08:53:13 +0000 Subject: [PATCH 01/25] Replace coordinator artifact roots with task_handler_bundle_name ADR-0013 locates Lang-SDK task handlers through a named Dag bundle, so the unversioned jars_root, executables_root and bundles_root directories go away instead of living beside it. dag_bundle_name becomes task_handler_bundle_name because the native-Dag work will use the old name with a different meaning. It has not shipped in a release, so there is no shim. A coordinator reads handlers from the named bundle at its current version, or from the task's own Dag bundle at the run's version, pinned for the task either way. CoordinatorManager.from_config validates the name of every spec without constructing coordinators, so a typo fails at config load instead of on the first task. --- .../src/airflow/config_templates/config.yml | 10 +- .../sdk/coordinators/_bundle_metadata.py | 45 +---- .../airflow/sdk/coordinators/_subprocess.py | 131 +++--------- .../coordinators/executable/coordinator.py | 30 +-- .../sdk/coordinators/java/coordinator.py | 34 ++-- .../sdk/coordinators/node/coordinator.py | 31 +-- .../airflow/sdk/execution_time/coordinator.py | 18 +- .../executable/test_coordinator.py | 73 +++---- .../coordinators/java/test_coordinator.py | 92 ++++----- .../coordinators/node/test_coordinator.py | 45 +---- .../task_sdk/coordinators/test_subprocess.py | 190 +++++------------- .../execution_time/test_coordinator.py | 31 ++- 12 files changed, 232 insertions(+), 498 deletions(-) diff --git a/airflow-core/src/airflow/config_templates/config.yml b/airflow-core/src/airflow/config_templates/config.yml index 34849d5eb5e13..0d93997abe387 100644 --- a/airflow-core/src/airflow/config_templates/config.yml +++ b/airflow-core/src/airflow/config_templates/config.yml @@ -2191,6 +2191,12 @@ sdk: example, two ``JavaCoordinator`` instances pinned to different JDK versions). + The language SDK coordinators read their compiled task handlers from the + Dag bundle named by the ``task_handler_bundle_name`` kwarg. It must name a + bundle in ``[dag_processor] dag_bundle_config_list`` and is checked when + this option is loaded. When it is unset, a task reads its handlers from its + own Dag bundle. + An entry may also carry an optional ``extra`` object for additional information associated with the coordinator that the coordinator itself does not receive; other components read it as needed. For example, @@ -2205,7 +2211,7 @@ sdk: "jdk-17": { "classpath": "airflow.sdk.coordinators.java.JavaCoordinator", "kwargs": { - "jars_root": ["/opt/airflow/java-bundles"], + "task_handler_bundle_name": "java-task-handlers", "java_executable": "/usr/lib/jvm/java-17-openjdk/bin/java", "jvm_args": ["-Xmx1024m"] }, @@ -2218,7 +2224,7 @@ sdk: "go-sdk": { "classpath": "airflow.sdk.coordinators.executable.ExecutableCoordinator", "kwargs": { - "executables_root": ["/opt/airflow/executable-bundles"] + "task_handler_bundle_name": "go-task-handlers" } } } diff --git a/task-sdk/src/airflow/sdk/coordinators/_bundle_metadata.py b/task-sdk/src/airflow/sdk/coordinators/_bundle_metadata.py index ec6e6419cbab2..b97ed80e8d82e 100644 --- a/task-sdk/src/airflow/sdk/coordinators/_bundle_metadata.py +++ b/task-sdk/src/airflow/sdk/coordinators/_bundle_metadata.py @@ -19,10 +19,9 @@ from __future__ import annotations -import os import pathlib import stat -from typing import TYPE_CHECKING, Any, Final +from typing import TYPE_CHECKING, Any import attrs import structlog @@ -38,24 +37,6 @@ log: FilteringBoundLogger = structlog.get_logger(logger_name="coordinators") -class _UnsetArtifactRoots: - """Sentinel distinguishing an omitted artifact-roots option from an empty value.""" - - -ARTIFACT_ROOTS_NOT_CONFIGURED: Final = _UnsetArtifactRoots() - - -def convert_roots( - value: None | os.PathLike[str] | pathlib.Path | list[os.PathLike[str] | pathlib.Path], -) -> list[pathlib.Path]: - """Normalize a coordinator's root-directories kwarg into a list of expanded paths.""" - if value is None: - return [] - if isinstance(value, (str, os.PathLike, pathlib.Path)): - return [pathlib.Path(value).expanduser()] - return [pathlib.Path(v).expanduser() for v in value] - - def walk_files( roots: Iterable[pathlib.Path], *, match: Callable[[pathlib.Path], bool] ) -> Iterator[pathlib.Path]: @@ -103,30 +84,6 @@ def _sorted_children(directory: pathlib.Path) -> list[pathlib.Path]: return [] -def convert_configured_roots( - value: _UnsetArtifactRoots - | None - | os.PathLike[str] - | pathlib.Path - | list[os.PathLike[str] | pathlib.Path], -) -> list[pathlib.Path]: - """Normalize configured roots while rejecting explicitly empty values.""" - if isinstance(value, _UnsetArtifactRoots): - return [] - if isinstance(value, (str, os.PathLike)): - values = [value] - elif value: - values = value - else: - values = [] - if not values or any(not os.fspath(v).strip() for v in values): - raise ValueError( - "Artifact roots must contain at least one path when provided, and each path must be non-empty; " - "omit the option to use the task's Dag bundle." - ) - return convert_roots(values) - - def validate_schema_version(instance, _, value) -> str: """Attrs validator resolving a bundle's supervisor schema version to a known one.""" return get_schema_version_migrator().resolve_version(str(value)) diff --git a/task-sdk/src/airflow/sdk/coordinators/_subprocess.py b/task-sdk/src/airflow/sdk/coordinators/_subprocess.py index 9e69255ba3d07..7ddddc9eeca4e 100644 --- a/task-sdk/src/airflow/sdk/coordinators/_subprocess.py +++ b/task-sdk/src/airflow/sdk/coordinators/_subprocess.py @@ -28,7 +28,6 @@ from __future__ import annotations import contextlib -import enum import ipaddress import itertools import os @@ -44,7 +43,6 @@ import structlog from airflow.dag_processing.bundles.base import BundleVersionLock, unpack_bundle_version # noqa: SDK002 -from airflow.dag_processing.bundles.manager import DagBundlesManager # noqa: SDK002 from airflow.sdk.api.datamodels._generated import BundleInfo from airflow.sdk.configuration import conf from airflow.sdk.execution_time.bundles import initialize_ti_bundle @@ -402,22 +400,6 @@ def _initialize_pinned_bundle(target: BundleInfo, logger: FilteringBoundLogger) return initialize_ti_bundle(BundleInfo(name=target.name, version=version, version_data=version_data)) -class _ArtifactSource(enum.Enum): - """How a subprocess coordinator locates the compiled task artifacts.""" - - EXPLICIT_ROOT = enum.auto() - """An explicit filesystem root (``jars_root`` / ``executables_root`` / ``bundles_root``).""" - NAMED_BUNDLE = enum.auto() - """``dag_bundle_name`` names a configured Dag bundle; its version current at task start is used.""" - TASK_BUNDLE = enum.auto() - """ - Neither is set: artifacts are *co-located* with the Python stub Dag. - - The task's own bundle is scanned, so the compiled artifacts ship in the same - bundle, at the same version, as the Dag that delegates to them. - """ - - @attrs.define(kw_only=True) class SubprocessCoordinator(BaseCoordinator): """ @@ -432,97 +414,38 @@ class SubprocessCoordinator(BaseCoordinator): :param task_startup_timeout: Maximum time the coordinator waits for the subprocess to connect to both servers, in seconds. The default is 10 seconds. - :param dag_bundle_name: Locate artifacts through a configured Dag bundle rather - than an explicit root. Mutually exclusive with the subclass's explicit root; - if neither is set, the task's own bundle is used. A named bundle resolves to - the version current when the task starts; the task's own bundle uses the - run's version. Either way the resolved version is pinned for the whole task. + :param task_handler_bundle_name: Name of the Dag bundle that holds the compiled + task handlers. It must be registered in ``[dag_processor] dag_bundle_config_list``. + If unset, the task's own Dag bundle is used. A named bundle resolves to the + version current when the task starts; the task's own bundle uses the run's + version. Either way the resolved version is pinned for the whole task. """ task_startup_timeout: float = 10.0 - dag_bundle_name: str | None = None + task_handler_bundle_name: str | None = None - _artifact_source: _ArtifactSource = attrs.field(init=False) - # The subclass's explicit root, recorded at construction so the base can - # resolve roots without knowing the subclass field name. - _configured_roots: list[pathlib.Path] = attrs.field(init=False, factory=list) _active_scan_roots: tuple[pathlib.Path, ...] | None = attrs.field(init=False, default=None) - @property - def _explicit_artifact_roots(self) -> tuple[str, Sequence[pathlib.Path]]: - """ - The subclass's explicit-root kwarg name and its configured value. - - The name is only used in error messages. An empty value — the default, for a - subclass that does not override this — selects task-bundle mode rather than - failing at execute time. - """ - return "root", () - - def __attrs_post_init__(self) -> None: - self._classify_artifact_source() - - def _classify_artifact_source(self) -> None: - """ - Classify and validate how this coordinator locates artifacts (construction time). - - Rejects setting both an explicit root and ``dag_bundle_name``, fails fast - when ``dag_bundle_name`` names a bundle that is not configured, and records - the resulting :class:`_ArtifactSource` and explicit root. - """ - root_kwarg, configured = self._explicit_artifact_roots - if configured and self.dag_bundle_name is not None: - raise ValueError( - f"Set at most one of {root_kwarg!r} or 'dag_bundle_name': {root_kwarg!r} for an " - f"explicit path, 'dag_bundle_name' for a configured Dag bundle, or leave both " - f"unset to scan the task's own bundle." - ) - if configured: - source = _ArtifactSource.EXPLICIT_ROOT - self._configured_roots = list(configured) - elif self.dag_bundle_name is not None: - source = _ArtifactSource.NAMED_BUNDLE - if not DagBundlesManager.is_bundle_configured(self.dag_bundle_name): - raise ValueError( - f"Coordinator 'dag_bundle_name' references unconfigured Dag bundle " - f"{self.dag_bundle_name!r}." - ) - else: - source = _ArtifactSource.TASK_BUNDLE - - self._artifact_source = source - log.debug( - "Coordinator artifact source selected", - mode=source.name, - dag_bundle_name=self.dag_bundle_name, - configured_roots=[str(root) for root in self._configured_roots], - ) - - def _init_root_source( + def _resolve_artifact_bundle( self, bundle_info: BundleInfo, logger: FilteringBoundLogger - ) -> tuple[list[pathlib.Path], BaseDagBundle | None]: + ) -> BaseDagBundle: """ - Resolve the directories to scan for artifacts, dispatched on the classified mode. + Materialize the Dag bundle holding the artifacts for a task of *bundle_info*. - Returns ``(roots, bundle)``: an explicit root yields no bundle (``None``); - a Dag-bundle mode returns the materialized path and the resolved bundle so - :meth:`execute_task` can hold a version lock over it. *logger* is the task - logger, so materialization failures surface in the task log. + That is the bundle named by ``task_handler_bundle_name``, or the task's own + bundle when it is unset. *logger* is the task logger, so materialization + failures surface in the task log. """ - if self._artifact_source is _ArtifactSource.EXPLICIT_ROOT: - return self._configured_roots, None - - if self._artifact_source is _ArtifactSource.NAMED_BUNDLE: - # NAMED_BUNDLE implies dag_bundle_name is set. - target = BundleInfo(name=cast("str", self.dag_bundle_name)) - else: + if self.task_handler_bundle_name is None: target = bundle_info + else: + target = BundleInfo(name=self.task_handler_bundle_name) bundle = _initialize_pinned_bundle(target, logger) path = bundle.path if not path.exists(): raise FileNotFoundError(f"Dag bundle {target.name!r} resolved to {path}, which does not exist.") - return [path], bundle + return bundle def _get_scan_roots(self) -> tuple[pathlib.Path, ...]: """Return the artifact roots resolved for the active task.""" @@ -567,20 +490,14 @@ def execute_task( subprocess_logs_to_stdout: bool, **kwargs, ) -> BaseCoordinator.ExecutionResult: - task_logger = logger or log - with contextlib.ExitStack() as stack: - roots, resolved_bundle = self._init_root_source(bundle_info, task_logger) - if resolved_bundle is not None: - # Hold the version lock across start()/wait() so bundle cleanup - # cannot rmtree a version this task is still reading from, - # mirroring task_runner.main() for the Python task path. - stack.enter_context( - BundleVersionLock( - bundle_name=resolved_bundle.name, - bundle_version=resolved_bundle.version, - ) - ) - stack.enter_context(self._set_scan_roots(roots)) + bundle = self._resolve_artifact_bundle(bundle_info, logger or log) + # Hold the version lock across start()/wait() so bundle cleanup cannot + # rmtree a version this task is still reading from, mirroring + # task_runner.main() for the Python task path. + with ( + BundleVersionLock(bundle_name=bundle.name, bundle_version=bundle.version), + self._set_scan_roots([bundle.path]), + ): command, subprocess_schema_version = self._build_execute_task_command(what=what) process = _PopenActivitySubprocess.start( what=what, diff --git a/task-sdk/src/airflow/sdk/coordinators/executable/coordinator.py b/task-sdk/src/airflow/sdk/coordinators/executable/coordinator.py index 9d6b80b249f81..237cc8dbbdf90 100644 --- a/task-sdk/src/airflow/sdk/coordinators/executable/coordinator.py +++ b/task-sdk/src/airflow/sdk/coordinators/executable/coordinator.py @@ -31,9 +31,7 @@ import structlog from airflow.sdk.coordinators._bundle_metadata import ( - ARTIFACT_ROOTS_NOT_CONFIGURED, ResolvedBundle, - convert_configured_roots, extract_supervisor_schema_version, parse_metadata_mapping, ) @@ -294,10 +292,10 @@ def _walk_executables( @attrs.define class _Bundle(ResolvedBundle): @classmethod - def find(cls, executables_root: Sequence[pathlib.Path], dag_id: str) -> Self: - log.debug("Finding executable bundles recursively", roots=executables_root) + def find(cls, roots: Sequence[pathlib.Path], dag_id: str) -> Self: + log.debug("Finding executable bundles recursively", roots=roots) rejected: list[tuple[pathlib.Path, str]] = [] - for p in _find_executables(executables_root): + for p in _find_executables(roots): if (metadata := _read_bundle_metadata(p)) is None: continue if dag_id not in _dag_ids(metadata): @@ -310,7 +308,7 @@ def find(cls, executables_root: Sequence[pathlib.Path], dag_id: str) -> Self: rejected.append((p.resolve(), str(exc))) continue - resolved_paths = os.pathsep.join(str(r.resolve()) for r in executables_root) + resolved_paths = os.pathsep.join(str(r.resolve()) for r in roots) if rejected: details = "; ".join(f"{path}: {reason}" for path, reason in rejected) tp = ( @@ -334,27 +332,19 @@ class ExecutableCoordinator(SubprocessCoordinator): "go": { "classpath": "airflow.sdk.coordinators.executable.ExecutableCoordinator", "kwargs": { - "executables_root": ["~/airflow/executable-bundles"] + "task_handler_bundle_name": "go-task-handlers" } } - :param executables_root: A list of directories scanned for executable - bundles when a Python stub DAG delegates task execution to a native - runtime. See :class:`SubprocessCoordinator` for its interaction with - ``dag_bundle_name``. + :param task_handler_bundle_name: Name of the Dag bundle holding the + executable bundles a Python stub Dag delegates task execution to. It must + be registered in ``[dag_processor] dag_bundle_config_list``. If unset, + the task's own Dag bundle is used. Only files with the executable bit set + are considered. :param task_startup_timeout: Maximum time the coordinator waits for a task process to start, in seconds. The default is 10 seconds. """ - executables_root: list[pathlib.Path] = attrs.field( - default=ARTIFACT_ROOTS_NOT_CONFIGURED, - converter=convert_configured_roots, - ) - - @property - def _explicit_artifact_roots(self) -> tuple[str, list[pathlib.Path]]: - return "executables_root", self.executables_root - def _build_execute_task_command(self, *, what: TaskInstance) -> tuple[list[str], str | None]: roots = self._get_scan_roots() bundle = _Bundle.find(roots, what.dag_id) diff --git a/task-sdk/src/airflow/sdk/coordinators/java/coordinator.py b/task-sdk/src/airflow/sdk/coordinators/java/coordinator.py index 1f0f6e6efd5f9..5d483110580f2 100644 --- a/task-sdk/src/airflow/sdk/coordinators/java/coordinator.py +++ b/task-sdk/src/airflow/sdk/coordinators/java/coordinator.py @@ -29,11 +29,7 @@ import attrs import structlog -from airflow.sdk.coordinators._bundle_metadata import ( - ARTIFACT_ROOTS_NOT_CONFIGURED, - convert_configured_roots, - validate_schema_version, -) +from airflow.sdk.coordinators._bundle_metadata import validate_schema_version from airflow.sdk.coordinators._subprocess import SubprocessCoordinator if TYPE_CHECKING: @@ -86,8 +82,8 @@ def _iter_dir(directory: pathlib.Path) -> Iterator[pathlib.Path]: return -def _calculate_classpath(jars_root: Sequence[pathlib.Path]) -> str: - jars = (p.as_posix() for p in _find_jars(jars_root)) +def _calculate_classpath(roots: Sequence[pathlib.Path]) -> str: + jars = (p.as_posix() for p in _find_jars(roots)) return os.pathsep.join(sorted(jars)) # Keep output deterministic. @@ -167,7 +163,7 @@ class JavaCoordinator(SubprocessCoordinator): "jdk-17": { "classpath": "airflow.sdk.coordinators.java.JavaCoordinator", "kwargs": { - "jars_root": ["~/airflow/jars"], + "task_handler_bundle_name": "java-task-handlers", "java_executable": "/usr/lib/jvm/java-17-openjdk/bin/java", "jvm_args": ["-Xmx1024m"] } @@ -176,16 +172,22 @@ class JavaCoordinator(SubprocessCoordinator): :param java_executable: Path to the ``java`` command (defaults to ``"java"``, which relies on ``$PATH``). :param jvm_args: Extra arguments passed to the JVM (e.g. ``["-Xmx512m"]``). - :param jars_root: A list of directories scanned for JAR bundles. See - :class:`SubprocessCoordinator` for its interaction with ``dag_bundle_name``. + :param task_handler_bundle_name: Name of the Dag bundle holding the JARs. It + must be registered in ``[dag_processor] dag_bundle_config_list``. If + unset, the task's own Dag bundle is used. :param main_class: Explicit entry point to execute with *java_executable*. :param task_startup_timeout: Maximum time the coordinator waits for a task process to start, in seconds. The default is 10 seconds. - If *main_class* is not explicitly set, JavaCoordinator scans *jars_root* to + Every JAR in the bundle goes on the classpath, so one bundle is one + classpath. Handlers that need conflicting dependency versions belong in + separate bundles, each served by its own coordinator and queue. + + If *main_class* is not explicitly set, JavaCoordinator scans the bundle to find an executable JAR (one with Main-Class set in its metadata). If more than one executable JAR is found, it may be nondeterministic which one ends - up being executed. + up being executed, so set *main_class* when more than one JAR in the bundle + declares Main-Class. A JAR containing metadata *Airflow-Supervisor-Schema-Version* should also be available to specify the wire schema version. The JAR containing the Java @@ -203,16 +205,8 @@ class JavaCoordinator(SubprocessCoordinator): java_executable: str = "java" jvm_args: list[str] = attrs.field(factory=list) - jars_root: list[pathlib.Path] = attrs.field( - default=ARTIFACT_ROOTS_NOT_CONFIGURED, - converter=convert_configured_roots, - ) main_class: str = "" - @property - def _explicit_artifact_roots(self) -> tuple[str, list[pathlib.Path]]: - return "jars_root", self.jars_root - def _build_execute_task_command(self, *, what: TaskInstance) -> tuple[list[str], str | None]: # Without main_class, the first executable JAR in walk order wins; tracked at # https://github.com/apache/airflow/issues/71134 diff --git a/task-sdk/src/airflow/sdk/coordinators/node/coordinator.py b/task-sdk/src/airflow/sdk/coordinators/node/coordinator.py index dd2a3f6f0416f..27fc142a3036c 100644 --- a/task-sdk/src/airflow/sdk/coordinators/node/coordinator.py +++ b/task-sdk/src/airflow/sdk/coordinators/node/coordinator.py @@ -26,12 +26,7 @@ import attrs import structlog -from airflow.sdk.coordinators._bundle_metadata import ( - ARTIFACT_ROOTS_NOT_CONFIGURED, - ResolvedBundle, - convert_configured_roots, - walk_files, -) +from airflow.sdk.coordinators._bundle_metadata import ResolvedBundle, walk_files from airflow.sdk.coordinators._subprocess import SubprocessCoordinator from airflow.sdk.coordinators.node._bundle_reader import read_bundle @@ -55,11 +50,11 @@ def _is_bundle(path: pathlib.Path) -> bool: @attrs.define class _Bundle(ResolvedBundle): @classmethod - def find(cls, bundles_root: Sequence[pathlib.Path], dag_id: str) -> Self: + def find(cls, roots: Sequence[pathlib.Path], dag_id: str) -> Self: """Return the first verified configured bundle that declares *dag_id*.""" - log.debug("Finding TypeScript bundles recursively", roots=bundles_root, dag_id=dag_id) + log.debug("Finding TypeScript bundles recursively", roots=roots, dag_id=dag_id) rejected: list[tuple[pathlib.Path, str]] = [] - for candidate in walk_files(bundles_root, match=_is_bundle): + for candidate in walk_files(roots, match=_is_bundle): try: metadata = read_bundle(candidate) if dag_id not in metadata.dag_ids: @@ -85,7 +80,7 @@ def find(cls, bundles_root: Sequence[pathlib.Path], dag_id: str) -> Self: log.debug("Selected TypeScript bundle", path=candidate, dag_id=dag_id) return bundle - searched = os.pathsep.join(os.fspath(root) for root in bundles_root) + searched = os.pathsep.join(os.fspath(root) for root in roots) if rejected: details = "; ".join(f"{path}: {reason}" for path, reason in rejected) raise FileNotFoundError( @@ -108,29 +103,21 @@ class NodeCoordinator(SubprocessCoordinator): "classpath": "airflow.sdk.coordinators.node.NodeCoordinator", "kwargs": { "node_executable": "node", - "bundles_root": ["/opt/airflow/ts-bundles"], + "task_handler_bundle_name": "ts-task-handlers", }, } } :param node_executable: Path to the ``node`` binary (defaults to ``"node"``, which relies on ``$PATH``). - :param bundles_root: Directories searched recursively, in order, for the first verified - ``*.min.mjs`` bundle declaring the task instance's Dag. See :class:`SubprocessCoordinator` - for its interaction with ``dag_bundle_name``. + :param task_handler_bundle_name: Name of the Dag bundle searched recursively for the first + verified ``*.min.mjs`` bundle declaring the task instance's Dag. It must be registered in + ``[dag_processor] dag_bundle_config_list``. If unset, the task's own Dag bundle is used. :param task_startup_timeout: Maximum time the coordinator waits for a task process to start, in seconds. The default is 10 seconds. """ node_executable: str = "node" - bundles_root: list[pathlib.Path] = attrs.field( - default=ARTIFACT_ROOTS_NOT_CONFIGURED, - converter=convert_configured_roots, - ) - - @property - def _explicit_artifact_roots(self) -> tuple[str, list[pathlib.Path]]: - return "bundles_root", self.bundles_root def _build_execute_task_command(self, *, what: TaskInstance) -> tuple[list[str], str | None]: roots = self._get_scan_roots() diff --git a/task-sdk/src/airflow/sdk/execution_time/coordinator.py b/task-sdk/src/airflow/sdk/execution_time/coordinator.py index 1469b27888c51..8aad5c6899dbc 100644 --- a/task-sdk/src/airflow/sdk/execution_time/coordinator.py +++ b/task-sdk/src/airflow/sdk/execution_time/coordinator.py @@ -48,6 +48,7 @@ import pydantic import structlog +from airflow.dag_processing.bundles.manager import DagBundlesManager # noqa: SDK002 from airflow.sdk._shared.module_loading import import_string from airflow.sdk.configuration import conf @@ -253,7 +254,13 @@ class CoordinatorManager: @classmethod def from_config(cls) -> Self: - """Load coordinator specs from configuration without initialization.""" + """ + Load coordinator specs from configuration without initialization. + + Every ``queue_to_coordinator`` key and every ``task_handler_bundle_name`` + kwarg is validated here, so a typo fails at config load rather than on the + first task routed to the coordinator. + """ coordinator_specs = { k: _CoordinatorSpec.model_validate(v) for k, v in conf.getjson("sdk", "coordinators", fallback={}).items() @@ -262,6 +269,15 @@ def from_config(cls) -> Self: for key in queue_to_coordinator.values(): if key not in coordinator_specs: raise ValueError(f"[sdk] queue_to_coordinator references invalid coordinator key: {key!r}") + for key, spec in coordinator_specs.items(): + bundle_name = spec.kwargs.get("task_handler_bundle_name") + if bundle_name is None: + continue + if not isinstance(bundle_name, str) or not DagBundlesManager.is_bundle_configured(bundle_name): + raise InvalidCoordinatorError( + f"[sdk] coordinators {key!r} sets task_handler_bundle_name={bundle_name!r}, " + f"which is not a bundle in [dag_processor] dag_bundle_config_list" + ) return cls(coordinator_specs=coordinator_specs, queue_to_coordinator=queue_to_coordinator) def _find_queue(self, key: str) -> BaseCoordinator: diff --git a/task-sdk/tests/task_sdk/coordinators/executable/test_coordinator.py b/task-sdk/tests/task_sdk/coordinators/executable/test_coordinator.py index c20913e26eae3..7071c9134bfb9 100644 --- a/task-sdk/tests/task_sdk/coordinators/executable/test_coordinator.py +++ b/task-sdk/tests/task_sdk/coordinators/executable/test_coordinator.py @@ -18,7 +18,7 @@ from __future__ import annotations import hashlib -import pathlib +import json import socket import stat import struct @@ -43,6 +43,7 @@ from airflow.sdk.execution_time.coordinator import BaseCoordinator from airflow.sdk.execution_time.supervisor import ActivitySubprocess +from tests_common.test_utils.config import conf_vars from tests_common.test_utils.version_compat import AIRFLOW_V_3_3_PLUS if not AIRFLOW_V_3_3_PLUS: @@ -361,46 +362,6 @@ def test_logs_when_metadata_is_not_a_mapping(self, tmp_path): class TestExecutableCoordinatorAttributes: - def test_executables_root_accepts_single_path(self, tmp_path): - coordinator = ExecutableCoordinator(executables_root=str(tmp_path)) - assert coordinator.executables_root == [tmp_path] - - def test_executables_root_accepts_list(self, tmp_path): - other = tmp_path / "other" - coordinator = ExecutableCoordinator(executables_root=[str(tmp_path), other]) - assert coordinator.executables_root == [tmp_path, other] - - def test_executables_root_optional_defaults_to_empty(self): - coordinator = ExecutableCoordinator() - assert coordinator.executables_root == [] - assert coordinator.dag_bundle_name is None - - @pytest.mark.parametrize( - "executables_root", - [None, [], "", " ", [""]], - ids=["none", "empty-list", "empty-str", "blank-str", "list-of-empty-str"], - ) - def test_explicit_empty_executables_root_raises(self, executables_root): - with pytest.raises(ValueError, match="and each path must be non-empty"): - ExecutableCoordinator(executables_root=executables_root) - - def test_root_and_dag_bundle_name_are_mutually_exclusive(self, tmp_path): - with pytest.raises(ValueError, match="at most one of 'executables_root' or 'dag_bundle_name'"): - ExecutableCoordinator(executables_root=[tmp_path], dag_bundle_name="artifacts") - - @patch("airflow.sdk.coordinators._subprocess.DagBundlesManager") - def test_unconfigured_dag_bundle_name_raises(self, mock_manager): - mock_manager.is_bundle_configured.return_value = False - with pytest.raises(ValueError, match="unconfigured Dag bundle 'ghost'"): - ExecutableCoordinator(dag_bundle_name="ghost") - - @patch("airflow.sdk.coordinators._subprocess.DagBundlesManager") - def test_configured_dag_bundle_name_accepted(self, mock_manager): - mock_manager.is_bundle_configured.return_value = True - coordinator = ExecutableCoordinator(dag_bundle_name="artifacts") - assert coordinator.dag_bundle_name == "artifacts" - mock_manager.is_bundle_configured.assert_called_once_with("artifacts") - def test_build_command_scans_passed_roots_in_colocated_mode(self, tmp_path): binary = _build_bundle(tmp_path / "my_bundle", dag_ids=["tutorial_dag"]) coordinator = ExecutableCoordinator() @@ -417,7 +378,7 @@ def test_returns_resolved_executable_and_schema_version(self, tmp_path): binary = _build_bundle(tmp_path / "my_bundle", dag_ids=["tutorial_dag"]) ti = _make_ti(dag_id="tutorial_dag") - coordinator = ExecutableCoordinator(executables_root=[tmp_path]) + coordinator = ExecutableCoordinator() with coordinator._set_scan_roots([tmp_path]): command, schema_version = coordinator._build_execute_task_command(what=ti) assert command == [str(binary.resolve())] @@ -429,7 +390,7 @@ def test_raises_when_bundle_omits_schema_version(self, tmp_path): _build_bundle(tmp_path / "my_bundle", dag_ids=["tutorial_dag"], metadata=metadata) ti = _make_ti(dag_id="tutorial_dag") - coordinator = ExecutableCoordinator(executables_root=[tmp_path]) + coordinator = ExecutableCoordinator() with ( coordinator._set_scan_roots([tmp_path]), pytest.raises(FileNotFoundError, match="matching bundles were rejected"), @@ -440,7 +401,7 @@ def test_raises_when_dag_id_not_found(self, tmp_path): _build_bundle(tmp_path / "my_bundle", dag_ids=["other_dag"]) ti = _make_ti(dag_id="tutorial_dag") - coordinator = ExecutableCoordinator(executables_root=[tmp_path]) + coordinator = ExecutableCoordinator() with ( coordinator._set_scan_roots([tmp_path]), pytest.raises(FileNotFoundError, match="cannot find executable bundle"), @@ -454,6 +415,18 @@ def bundles_dir(tmp_path): return tmp_path +@pytest.fixture +def go_task_handlers(bundles_dir): + """Register *bundles_dir* as the ``go-task-handlers`` Dag bundle and return that name.""" + bundle = { + "name": "go-task-handlers", + "classpath": "airflow.dag_processing.bundles.local.LocalDagBundle", + "kwargs": {"path": str(bundles_dir)}, + } + with conf_vars({("dag_processor", "dag_bundle_config_list"): json.dumps([bundle])}): + yield bundle["name"] + + @pytest.fixture def mock_client(make_ti_context): client = MagicMock() @@ -462,10 +435,10 @@ def mock_client(make_ti_context): class TestExecutableCoordinatorExecuteTask: - def _captured_popen_cmd(self, bundles_dir: pathlib.Path, mock_client) -> list[str]: + def _captured_popen_cmd(self, bundle_name: str, mock_client) -> list[str]: """Run execute_task with mocked subprocess and return the command list.""" ti = _make_ti(dag_id="tutorial_dag") - coordinator = ExecutableCoordinator(executables_root=[bundles_dir]) + coordinator = ExecutableCoordinator(task_handler_bundle_name=bundle_name) mock_proc = MagicMock(spec=subprocess.Popen) mock_proc.pid = 12345 @@ -505,14 +478,14 @@ def capture_popen(cmd, **kwargs): assert popen_calls, "subprocess.Popen was not called" return popen_calls[0] - def test_executable_path_is_first_arg(self, bundles_dir, mock_client): - cmd = self._captured_popen_cmd(bundles_dir, mock_client) + def test_executable_path_is_first_arg(self, bundles_dir, go_task_handlers, mock_client): + cmd = self._captured_popen_cmd(go_task_handlers, mock_client) expected = str((bundles_dir / "my_bundle").resolve()) assert cmd[0] == expected - def test_returns_execution_result(self, bundles_dir, mock_client): + def test_returns_execution_result(self, go_task_handlers, mock_client): ti = _make_ti(dag_id="tutorial_dag") - coordinator = ExecutableCoordinator(executables_root=[bundles_dir]) + coordinator = ExecutableCoordinator(task_handler_bundle_name=go_task_handlers) mock_proc = MagicMock(spec=subprocess.Popen) mock_proc.pid = 99999 diff --git a/task-sdk/tests/task_sdk/coordinators/java/test_coordinator.py b/task-sdk/tests/task_sdk/coordinators/java/test_coordinator.py index c55a2cc891253..eda377e6251fd 100644 --- a/task-sdk/tests/task_sdk/coordinators/java/test_coordinator.py +++ b/task-sdk/tests/task_sdk/coordinators/java/test_coordinator.py @@ -18,6 +18,7 @@ from __future__ import annotations +import json import os import pathlib import re @@ -39,6 +40,7 @@ from airflow.sdk.execution_time.coordinator import BaseCoordinator from airflow.sdk.execution_time.supervisor import ActivitySubprocess +from tests_common.test_utils.config import conf_vars from tests_common.test_utils.version_compat import AIRFLOW_V_3_3_PLUS if not AIRFLOW_V_3_3_PLUS: @@ -222,50 +224,22 @@ def test_skip_logged_when_directory_revisited(self, tmp_path): class TestJavaCoordinatorAttributes: def test_default_kwargs(self): - coordinator = JavaCoordinator(jars_root="/airflow/java-bundles") + coordinator = JavaCoordinator() assert coordinator.java_executable == "java" assert coordinator.jvm_args == [] - assert coordinator.jars_root == [pathlib.Path("/airflow/java-bundles")] + assert coordinator.task_handler_bundle_name is None + # main_class stays optional: the entrypoint is auto-detected from the bundle scan. + assert coordinator.main_class == "" def test_custom_kwargs(self): coordinator = JavaCoordinator( java_executable="/opt/java/bin/java", jvm_args=["-Xmx512m", "-Xms256m"], - jars_root=["/airflow/java-bundles"], + task_handler_bundle_name="java-task-handlers", ) assert coordinator.java_executable == "/opt/java/bin/java" assert coordinator.jvm_args == ["-Xmx512m", "-Xms256m"] - assert coordinator.jars_root == [pathlib.Path("/airflow/java-bundles")] - - def test_jars_root_optional_defaults_to_empty(self): - # main_class stays optional: the entrypoint is auto-detected from the bundle scan. - coordinator = JavaCoordinator() - assert coordinator.jars_root == [] - assert coordinator.dag_bundle_name is None - assert coordinator.main_class == "" - - @pytest.mark.parametrize( - "jars_root", - [None, [], "", " ", [""]], - ids=["none", "empty-list", "empty-str", "blank-str", "list-of-empty-str"], - ) - def test_explicit_empty_jars_root_raises(self, jars_root): - with pytest.raises(ValueError, match="and each path must be non-empty"): - JavaCoordinator(jars_root=jars_root) - - def test_explicit_root_does_not_require_main_class(self, tmp_path): - coordinator = JavaCoordinator(jars_root=tmp_path) - assert coordinator.main_class == "" - - def test_root_and_dag_bundle_name_are_mutually_exclusive(self): - with pytest.raises(ValueError, match="at most one of 'jars_root' or 'dag_bundle_name'"): - JavaCoordinator(jars_root="/airflow/java-bundles", dag_bundle_name="artifacts") - - @patch("airflow.sdk.coordinators._subprocess.DagBundlesManager") - def test_unconfigured_dag_bundle_name_raises(self, mock_manager): - mock_manager.is_bundle_configured.return_value = False - with pytest.raises(ValueError, match="unconfigured Dag bundle 'ghost'"): - JavaCoordinator(dag_bundle_name="ghost") + assert coordinator.task_handler_bundle_name == "java-task-handlers" def test_build_command_scans_passed_roots_in_colocated_mode(self, tmp_path): _make_jar(tmp_path / "app.jar", main_class="com.example.TaskRunner", schema_version="2026-06-16") @@ -283,6 +257,18 @@ def jars_root(tmp_path): return tmp_path +@pytest.fixture +def java_task_handlers(jars_root): + """Register *jars_root* as the ``java-task-handlers`` Dag bundle and return that name.""" + bundle = { + "name": "java-task-handlers", + "classpath": "airflow.dag_processing.bundles.local.LocalDagBundle", + "kwargs": {"path": str(jars_root)}, + } + with conf_vars({("dag_processor", "dag_bundle_config_list"): json.dumps([bundle])}): + yield bundle["name"] + + @pytest.fixture def mock_client(make_ti_context): client = MagicMock() @@ -293,7 +279,7 @@ def mock_client(make_ti_context): class TestJavaCoordinatorExecuteTask: def _captured_popen_cmd( self, - jars_root: pathlib.Path, + bundle_name: str, mock_client, *, java_executable: str = "java", @@ -304,7 +290,7 @@ def _captured_popen_cmd( coordinator = JavaCoordinator( java_executable=java_executable, jvm_args=jvm_args or [], - jars_root=jars_root, + task_handler_bundle_name=bundle_name, ) mock_proc = MagicMock(spec=subprocess.Popen) @@ -345,56 +331,58 @@ def capture_popen(cmd, **kwargs): assert popen_calls, "subprocess.Popen was not called" return popen_calls[0] - def test_java_executable_is_first_arg(self, jars_root, mock_client): + def test_java_executable_is_first_arg(self, java_task_handlers, mock_client): cmd = self._captured_popen_cmd( - jars_root, mock_client, java_executable="/usr/lib/jvm/java-17/bin/java" + java_task_handlers, mock_client, java_executable="/usr/lib/jvm/java-17/bin/java" ) assert cmd[0] == "/usr/lib/jvm/java-17/bin/java" - def test_classpath_flag_and_value_present(self, jars_root, mock_client): - cmd = self._captured_popen_cmd(jars_root, mock_client) + def test_classpath_flag_and_value_present(self, jars_root, java_task_handlers, mock_client): + cmd = self._captured_popen_cmd(java_task_handlers, mock_client) assert "-classpath" in cmd cp_idx = cmd.index("-classpath") classpath = cmd[cp_idx + 1] assert jars_root.joinpath("app.jar").as_posix() in classpath - def test_main_class_present(self, jars_root, mock_client): - cmd = self._captured_popen_cmd(jars_root, mock_client) + def test_main_class_present(self, java_task_handlers, mock_client): + cmd = self._captured_popen_cmd(java_task_handlers, mock_client) assert "com.example.TaskRunner" in cmd - def test_comm_and_logs_args_present(self, jars_root, mock_client): - cmd = self._captured_popen_cmd(jars_root, mock_client) + def test_comm_and_logs_args_present(self, java_task_handlers, mock_client): + cmd = self._captured_popen_cmd(java_task_handlers, mock_client) comm_args = [a for a in cmd if a.startswith("--comm=")] logs_args = [a for a in cmd if a.startswith("--logs=")] assert len(comm_args) == 1 assert len(logs_args) == 1 - def test_comm_and_logs_contain_port(self, jars_root, mock_client): - cmd = self._captured_popen_cmd(jars_root, mock_client) + def test_comm_and_logs_contain_port(self, java_task_handlers, mock_client): + cmd = self._captured_popen_cmd(java_task_handlers, mock_client) comm_arg = next(a for a in cmd if a.startswith("--comm=")) logs_arg = next(a for a in cmd if a.startswith("--logs=")) # format is host:port assert ":" in comm_arg.split("=", 1)[1] assert ":" in logs_arg.split("=", 1)[1] - def test_jvm_args_inserted_before_main_class(self, jars_root, mock_client): - cmd = self._captured_popen_cmd(jars_root, mock_client, jvm_args=["-Xmx512m", "-Dsome.prop=value"]) + def test_jvm_args_inserted_before_main_class(self, java_task_handlers, mock_client): + cmd = self._captured_popen_cmd( + java_task_handlers, mock_client, jvm_args=["-Xmx512m", "-Dsome.prop=value"] + ) main_idx = cmd.index("com.example.TaskRunner") for jvm_arg in ["-Xmx512m", "-Dsome.prop=value"]: assert jvm_arg in cmd assert cmd.index(jvm_arg) < main_idx - def test_comm_and_logs_after_main_class(self, jars_root, mock_client): - cmd = self._captured_popen_cmd(jars_root, mock_client) + def test_comm_and_logs_after_main_class(self, java_task_handlers, mock_client): + cmd = self._captured_popen_cmd(java_task_handlers, mock_client) main_idx = cmd.index("com.example.TaskRunner") comm_idx = next(i for i, a in enumerate(cmd) if a.startswith("--comm=")) logs_idx = next(i for i, a in enumerate(cmd) if a.startswith("--logs=")) assert comm_idx > main_idx assert logs_idx > main_idx - def test_returns_execution_result(self, jars_root, mock_client): + def test_returns_execution_result(self, java_task_handlers, mock_client): ti = _make_ti() - coordinator = JavaCoordinator(jars_root=jars_root) + coordinator = JavaCoordinator(task_handler_bundle_name=java_task_handlers) mock_proc = MagicMock(spec=subprocess.Popen) mock_proc.pid = 99999 diff --git a/task-sdk/tests/task_sdk/coordinators/node/test_coordinator.py b/task-sdk/tests/task_sdk/coordinators/node/test_coordinator.py index 146cb91c1d94c..3dc01be33f297 100644 --- a/task-sdk/tests/task_sdk/coordinators/node/test_coordinator.py +++ b/task-sdk/tests/task_sdk/coordinators/node/test_coordinator.py @@ -21,7 +21,6 @@ import json import pathlib from unittest import mock -from unittest.mock import patch import pytest from task_sdk.coordinators.node._bundle_test_utils import ( @@ -60,50 +59,22 @@ def _make_ti(dag_id: str = "test_dag", queue: str = "ts") -> TaskInstance: class TestNodeCoordinatorAttributes: def test_default_kwargs(self): - coordinator = NodeCoordinator(bundles_root="/airflow/ts-bundles") + coordinator = NodeCoordinator() assert coordinator.node_executable == "node" - assert coordinator.bundles_root == [pathlib.Path("/airflow/ts-bundles")] assert coordinator.task_startup_timeout == 10.0 def test_custom_kwargs(self): coordinator = NodeCoordinator( node_executable="/opt/node/bin/node", - bundles_root=["/airflow/ts-bundles", "~/extra-bundles"], + task_handler_bundle_name="ts-task-handlers", task_startup_timeout=30.0, ) assert coordinator.node_executable == "/opt/node/bin/node" - assert coordinator.bundles_root == [ - pathlib.Path("/airflow/ts-bundles"), - pathlib.Path("~/extra-bundles").expanduser(), - ] + assert coordinator.task_handler_bundle_name == "ts-task-handlers" assert coordinator.task_startup_timeout == 30.0 - def test_bundles_root_optional_defaults_to_empty(self): - coordinator = NodeCoordinator() - assert coordinator.bundles_root == [] - assert coordinator.dag_bundle_name is None - - @pytest.mark.parametrize( - "bundles_root", - [None, [], "", " ", [""]], - ids=["none", "empty-list", "empty-str", "blank-str", "list-of-empty-str"], - ) - def test_explicit_empty_bundles_root_raises(self, bundles_root): - with pytest.raises(ValueError, match="and each path must be non-empty"): - NodeCoordinator(bundles_root=bundles_root) - - def test_root_and_dag_bundle_name_are_mutually_exclusive(self): - with pytest.raises(ValueError, match="at most one of 'bundles_root' or 'dag_bundle_name'"): - NodeCoordinator(bundles_root="/airflow/ts-bundles", dag_bundle_name="artifacts") - - @patch("airflow.sdk.coordinators._subprocess.DagBundlesManager") - def test_unconfigured_dag_bundle_name_raises(self, mock_manager): - mock_manager.is_bundle_configured.return_value = False - with pytest.raises(ValueError, match="unconfigured Dag bundle 'ghost'"): - NodeCoordinator(dag_bundle_name="ghost") - def test_build_command_scans_passed_roots_in_colocated_mode(self, tmp_path): bundle = write_bundle(tmp_path, "test_dag") coordinator = NodeCoordinator() @@ -116,10 +87,7 @@ def test_build_command_scans_passed_roots_in_colocated_mode(self, tmp_path): class TestNodeCoordinatorExecuteTaskCommand: def test_selects_bundle_by_dag_id(self, tmp_path): selected = write_bundle(tmp_path, "sales") - coordinator = NodeCoordinator( - node_executable="/opt/node/bin/node", - bundles_root=tmp_path, - ) + coordinator = NodeCoordinator(node_executable="/opt/node/bin/node") with coordinator._set_scan_roots([tmp_path]): command, schema_version = coordinator._build_execute_task_command(what=_make_ti(dag_id="sales")) @@ -129,10 +97,7 @@ def test_selects_bundle_by_dag_id(self, tmp_path): def test_build_execute_task_command_returns_node_bundle_and_schema_version(self, tmp_path): bundle = write_bundle(tmp_path, "test_dag") - coordinator = NodeCoordinator( - node_executable="/opt/node/bin/node", - bundles_root=tmp_path, - ) + coordinator = NodeCoordinator(node_executable="/opt/node/bin/node") with coordinator._set_scan_roots([tmp_path]): command, schema_version = coordinator._build_execute_task_command(what=_make_ti()) diff --git a/task-sdk/tests/task_sdk/coordinators/test_subprocess.py b/task-sdk/tests/task_sdk/coordinators/test_subprocess.py index 5297f56dead4f..909a2c605f0ba 100644 --- a/task-sdk/tests/task_sdk/coordinators/test_subprocess.py +++ b/task-sdk/tests/task_sdk/coordinators/test_subprocess.py @@ -32,13 +32,12 @@ import pytest from uuid6 import uuid7 -from airflow.dag_processing.bundles.base import BundleVersion +from airflow.dag_processing.bundles.base import BaseDagBundle, BundleVersion from airflow.sdk.api.client import Client, TaskInstanceOperations from airflow.sdk.api.datamodels._generated import BundleInfo, TaskInstance from airflow.sdk.coordinators._subprocess import ( SubprocessCoordinator, _accept_connections, - _ArtifactSource, _connection_owned_by_process_tree, _is_connection_from_process, _PopenActivitySubprocess, @@ -573,26 +572,37 @@ def test_untrack_unknown_object_does_not_raise(self): class _StubSubprocessCoordinator(SubprocessCoordinator): """Minimal SubprocessCoordinator subclass used to exercise the base machinery. - ``explicit_roots`` defaults to a real path so the coordinator classifies as - EXPLICIT_ROOT and execute_task resolves without touching a Dag bundle; pass - ``explicit_roots=[]`` to exercise TASK_BUNDLE mode. Roots handed to the - command builder are recorded in ``recorded_roots`` so wiring can be asserted. + Roots handed to the command builder are recorded in ``recorded_roots`` so + wiring can be asserted. """ command: list[str] schema_version: str | None = None - explicit_roots: list[pathlib.Path] = attrs.field(factory=lambda: [pathlib.Path(".")]) recorded_roots: list[list[pathlib.Path]] = attrs.field(init=False, factory=list) - @property - def _explicit_artifact_roots(self) -> tuple[str, list[pathlib.Path]]: - return "explicit_roots", self.explicit_roots - def _build_execute_task_command(self, *, what): self.recorded_roots.append(list(self._get_scan_roots())) return list(self.command), self.schema_version +def _make_bundle(path: pathlib.Path, *, version: str | None = None, name: str = "dags") -> MagicMock: + bundle = MagicMock(spec=BaseDagBundle, path=path, version=version) + bundle.name = name + return bundle + + +@pytest.fixture +def artifact_bundle(tmp_path): + """Resolve every task's artifacts to an unversioned bundle at *tmp_path*, without the bundle machinery.""" + with patch.object( + SubprocessCoordinator, + "_resolve_artifact_bundle", + autospec=True, + return_value=_make_bundle(tmp_path), + ) as mock_resolve: + yield mock_resolve + + @pytest.fixture def mock_client(make_ti_context): client = MagicMock(spec=Client) @@ -618,6 +628,7 @@ class _Plain(SubprocessCoordinator): _Plain()._build_execute_task_command(what=_make_ti()) +@pytest.mark.usefixtures("artifact_bundle") class TestSubprocessCoordinatorExecuteTask: def _captured_popen_cmd( self, @@ -863,79 +874,20 @@ def test_register_pipe_readers_called_with_four_sockets(self, mock_client): assert mock_register.mock_calls == [call(ANY, ANY, ANY, ANY, data=ANY)] -class TestClassifyArtifactSource: - """Construction-time classification and per-mode validation. - - Classification runs from the base ``__attrs_post_init__``, so it is exercised - through construction rather than by calling the helper directly. - """ - - def test_explicit_root_classified_and_recorded(self): - configured = [pathlib.Path("/artifacts")] - coordinator = _StubSubprocessCoordinator(command=["x"], explicit_roots=configured) - assert coordinator._artifact_source is _ArtifactSource.EXPLICIT_ROOT - assert coordinator._configured_roots == configured - - def test_task_bundle_classified_when_neither_set(self): - coordinator = _StubSubprocessCoordinator(command=["x"], explicit_roots=[]) - assert coordinator._artifact_source is _ArtifactSource.TASK_BUNDLE - - def test_subclass_without_overrides_defaults_to_task_bundle(self): - """A subclass that overrides neither hook is classified, not left to error at execute time.""" - - @attrs.define(kw_only=True) - class _Bare(SubprocessCoordinator): - def _build_execute_task_command(self, *, what): - return [], None - - assert _Bare()._artifact_source is _ArtifactSource.TASK_BUNDLE - - def test_rejects_explicit_root_and_dag_bundle_name_together(self): - with pytest.raises(ValueError, match="at most one of 'explicit_roots' or 'dag_bundle_name'"): - _StubSubprocessCoordinator( - command=["x"], explicit_roots=[pathlib.Path("/artifacts")], dag_bundle_name="artifacts" - ) - - @patch("airflow.sdk.coordinators._subprocess.DagBundlesManager") - def test_rejects_unconfigured_dag_bundle_name(self, mock_manager): - mock_manager.is_bundle_configured.return_value = False - with pytest.raises(ValueError, match="unconfigured Dag bundle 'ghost'"): - _StubSubprocessCoordinator(command=["x"], explicit_roots=[], dag_bundle_name="ghost") - - @patch("airflow.sdk.coordinators._subprocess.DagBundlesManager") - def test_accepts_configured_dag_bundle_name(self, mock_manager): - mock_manager.is_bundle_configured.return_value = True - coordinator = _StubSubprocessCoordinator( - command=["x"], explicit_roots=[], dag_bundle_name="artifacts" - ) - assert coordinator._artifact_source is _ArtifactSource.NAMED_BUNDLE - mock_manager.is_bundle_configured.assert_called_once_with("artifacts") - - -class TestInitRootSource: - """Execute-time root resolution, dispatched on the classified mode.""" +class TestResolveArtifactBundle: + """Execute-time resolution of the Dag bundle holding the artifacts.""" - def test_returns_configured_root_and_no_bundle_in_explicit_mode(self): - configured = [pathlib.Path("/a"), pathlib.Path("/b")] - coordinator = _StubSubprocessCoordinator(command=["x"], explicit_roots=configured) - roots, bundle = coordinator._init_root_source(MagicMock(), log) - assert roots == configured - assert bundle is None - - @patch("airflow.sdk.coordinators._subprocess.initialize_ti_bundle") - def test_task_bundle_mode_uses_passed_task_bundle(self, mock_initialize, tmp_path): - resolved = MagicMock(path=tmp_path, version="v3") + @patch("airflow.sdk.coordinators._subprocess.initialize_ti_bundle", autospec=True) + def test_unset_name_uses_the_task_bundle(self, mock_initialize, tmp_path): + resolved = _make_bundle(tmp_path, version="v3") mock_initialize.return_value = resolved - coordinator = _StubSubprocessCoordinator(command=["x"], explicit_roots=[]) + coordinator = _StubSubprocessCoordinator(command=["x"]) bundle_info = BundleInfo(name="dags", version="v3", version_data={"k": "v"}) - roots, bundle = coordinator._init_root_source(bundle_info, log) - - assert roots == [tmp_path] - assert bundle is resolved + assert coordinator._resolve_artifact_bundle(bundle_info, log) is resolved mock_initialize.assert_called_once_with(bundle_info) - @patch("airflow.sdk.coordinators._subprocess.initialize_ti_bundle") + @patch("airflow.sdk.coordinators._subprocess.initialize_ti_bundle", autospec=True) def test_version_less_bundle_is_rematerialized_at_its_current_version(self, mock_initialize, tmp_path): """A bundle resolved without a version is re-resolved at the version current now. @@ -947,67 +899,52 @@ def test_version_less_bundle_is_rematerialized_at_its_current_version(self, mock pinned_tree = tmp_path / "versions" / "sha-abc" pinned_tree.mkdir(parents=True) - unpinned = MagicMock(path=shared_checkout, version=None) + unpinned = _make_bundle(shared_checkout) unpinned.get_current_version.return_value = BundleVersion(version="sha-abc", data={"k": "v"}) - pinned = MagicMock(path=pinned_tree, version="sha-abc") + pinned = _make_bundle(pinned_tree, version="sha-abc") mock_initialize.side_effect = [unpinned, pinned] - coordinator = _StubSubprocessCoordinator(command=["x"], explicit_roots=[]) - coordinator.dag_bundle_name = "artifacts" - coordinator._artifact_source = _ArtifactSource.NAMED_BUNDLE + coordinator = _StubSubprocessCoordinator(command=["x"], task_handler_bundle_name="artifacts") - roots, bundle = coordinator._init_root_source(MagicMock(), log) - - assert roots == [pinned_tree] - assert bundle is pinned + assert coordinator._resolve_artifact_bundle(MagicMock(spec=BundleInfo), log) is pinned assert mock_initialize.call_args_list == [ call(BundleInfo(name="artifacts")), call(BundleInfo(name="artifacts", version="sha-abc", version_data={"k": "v"})), ] - @patch("airflow.sdk.coordinators._subprocess.initialize_ti_bundle") + @patch("airflow.sdk.coordinators._subprocess.initialize_ti_bundle", autospec=True) def test_unversioned_bundle_keeps_its_single_path(self, mock_initialize, tmp_path): - resolved = MagicMock(path=tmp_path, version=None) + resolved = _make_bundle(tmp_path) resolved.get_current_version.return_value = None mock_initialize.return_value = resolved - coordinator = _StubSubprocessCoordinator(command=["x"], explicit_roots=[]) - coordinator.dag_bundle_name = "artifacts" - coordinator._artifact_source = _ArtifactSource.NAMED_BUNDLE - - roots, bundle = coordinator._init_root_source(MagicMock(), log) + coordinator = _StubSubprocessCoordinator(command=["x"], task_handler_bundle_name="artifacts") - assert roots == [tmp_path] - assert bundle is resolved + assert coordinator._resolve_artifact_bundle(MagicMock(spec=BundleInfo), log) is resolved mock_initialize.assert_called_once_with(BundleInfo(name="artifacts")) - @patch("airflow.sdk.coordinators._subprocess.initialize_ti_bundle") + @patch("airflow.sdk.coordinators._subprocess.initialize_ti_bundle", autospec=True) def test_missing_resolved_path_raises(self, mock_initialize, tmp_path): - missing = tmp_path / "nope" - mock_initialize.return_value = MagicMock(path=missing, version="v1") - coordinator = _StubSubprocessCoordinator(command=["x"], explicit_roots=[]) - coordinator.dag_bundle_name = "artifacts" - coordinator._artifact_source = _ArtifactSource.NAMED_BUNDLE + mock_initialize.return_value = _make_bundle(tmp_path / "nope", version="v1") + coordinator = _StubSubprocessCoordinator(command=["x"], task_handler_bundle_name="artifacts") with pytest.raises(FileNotFoundError, match="does not exist"): - coordinator._init_root_source(MagicMock(), log) + coordinator._resolve_artifact_bundle(MagicMock(spec=BundleInfo), log) class TestExecuteTaskBundleWiring: """execute_task passes the task bundle, forwards resolved roots, and holds the version lock.""" - @patch("airflow.sdk.coordinators._subprocess.BundleVersionLock") - @patch("airflow.sdk.coordinators._subprocess.initialize_ti_bundle") + @patch("airflow.sdk.coordinators._subprocess.BundleVersionLock", autospec=True) + @patch("airflow.sdk.coordinators._subprocess.initialize_ti_bundle", autospec=True) @patch.object(_PopenActivitySubprocess, "start") def test_task_bundle_mode_binds_forwards_roots_and_locks( self, mock_start, mock_initialize, mock_lock, mock_client, tmp_path ): - resolved = MagicMock(path=tmp_path, version="v9") - resolved.name = "dags" - mock_initialize.return_value = resolved + mock_initialize.return_value = _make_bundle(tmp_path, version="v9") mock_start.return_value.wait.return_value = 0 - coordinator = _StubSubprocessCoordinator(command=["/runtime"], explicit_roots=[]) + coordinator = _StubSubprocessCoordinator(command=["/runtime"]) bundle_info = BundleInfo(name="dags", version="v9") coordinator.execute_task( @@ -1024,27 +961,8 @@ def test_task_bundle_mode_binds_forwards_roots_and_locks( mock_lock.return_value.__enter__.assert_called_once() mock_lock.return_value.__exit__.assert_called_once() - @patch("airflow.sdk.coordinators._subprocess.BundleVersionLock") - @patch.object(_PopenActivitySubprocess, "start") - def test_explicit_root_mode_forwards_roots_without_locking( - self, mock_start, mock_lock, mock_client, tmp_path - ): - mock_start.return_value.wait.return_value = 0 - coordinator = _StubSubprocessCoordinator(command=["/runtime"], explicit_roots=[tmp_path]) - - coordinator.execute_task( - what=_make_ti(), - dag_rel_path="dag.py", - bundle_info=MagicMock(), - client=mock_client, - subprocess_logs_to_stdout=False, - ) - - assert coordinator.recorded_roots == [[tmp_path]] - mock_lock.assert_not_called() - - @patch("airflow.sdk.coordinators._subprocess.BundleVersionLock") - @patch("airflow.sdk.coordinators._subprocess.initialize_ti_bundle") + @patch("airflow.sdk.coordinators._subprocess.BundleVersionLock", autospec=True) + @patch("airflow.sdk.coordinators._subprocess.initialize_ti_bundle", autospec=True) @patch.object(_PopenActivitySubprocess, "start") def test_named_bundle_mode_locks_the_tree_it_scans( self, mock_start, mock_initialize, mock_lock, mock_client, tmp_path @@ -1053,21 +971,18 @@ def test_named_bundle_mode_locks_the_tree_it_scans( pinned_tree = tmp_path / "versions" / "sha-abc" pinned_tree.mkdir(parents=True) - unpinned = MagicMock(path=tmp_path, version=None) + unpinned = _make_bundle(tmp_path, name="artifacts") unpinned.get_current_version.return_value = BundleVersion(version="sha-abc", data=None) - pinned = MagicMock(path=pinned_tree, version="sha-abc") - pinned.name = "artifacts" + pinned = _make_bundle(pinned_tree, version="sha-abc", name="artifacts") mock_initialize.side_effect = [unpinned, pinned] mock_start.return_value.wait.return_value = 0 - coordinator = _StubSubprocessCoordinator(command=["/runtime"], explicit_roots=[]) - coordinator.dag_bundle_name = "artifacts" - coordinator._artifact_source = _ArtifactSource.NAMED_BUNDLE + coordinator = _StubSubprocessCoordinator(command=["/runtime"], task_handler_bundle_name="artifacts") coordinator.execute_task( what=_make_ti(), dag_rel_path="dag.py", - bundle_info=MagicMock(), + bundle_info=BundleInfo(name="dags", version="v9"), client=mock_client, subprocess_logs_to_stdout=False, ) @@ -1093,6 +1008,7 @@ def test_rejects_nested_reentry(self, tmp_path): with coordinator._set_scan_roots([tmp_path]): pass + @pytest.mark.usefixtures("artifact_bundle") def test_execute_task_is_not_reentrant(self, mock_client, tmp_path): coordinator = _StubSubprocessCoordinator(command=["/bin/true"]) with coordinator._set_scan_roots([tmp_path]): diff --git a/task-sdk/tests/task_sdk/execution_time/test_coordinator.py b/task-sdk/tests/task_sdk/execution_time/test_coordinator.py index c4185c1939092..3659bce784dbf 100644 --- a/task-sdk/tests/task_sdk/execution_time/test_coordinator.py +++ b/task-sdk/tests/task_sdk/execution_time/test_coordinator.py @@ -37,6 +37,8 @@ reset_coordinator_manager, ) +from tests_common.test_utils.config import conf_vars + class _CoordinatorA(BaseCoordinator): def __init__(self, *, label: str = "a"): @@ -134,6 +136,22 @@ def test_from_config_rejects_invalid_queue_mapping(self, sdk_config): ): CoordinatorManager.from_config() + @pytest.mark.parametrize("bundle_name", ["ghost", ["ghost"]], ids=["unconfigured", "not-a-string"]) + def test_from_config_rejects_invalid_task_handler_bundle_name(self, sdk_config, bundle_name): + """Validated for every spec, routed or not, without importing or constructing the coordinator.""" + sdk_config( + coordinators=json.dumps( + { + "boom": { + "classpath": f"{_ExplodingCoordinator.__module__}._ExplodingCoordinator", + "kwargs": {"task_handler_bundle_name": bundle_name}, + } + } + ), + ) + with pytest.raises(InvalidCoordinatorError, match=r"'boom' sets task_handler_bundle_name="): + CoordinatorManager.from_config() + @pytest.mark.parametrize( ("coordinator_spec", "expected_match"), [ @@ -261,8 +279,8 @@ def test_extra_for_queue_does_not_instantiate_coordinator(self, sdk_config): class TestConfigYamlCoordinatorsExample: """Guard the ``[sdk] coordinators`` example in ``config.yml`` against drift. - Nothing else exercises the example, so a broken one (e.g. dropping the - required ``jars_root`` kwarg) can ship unnoticed. Loading it through + Nothing else exercises the example, so a broken one (e.g. a kwarg the + coordinator does not accept) can ship unnoticed. Loading it through CoordinatorManager and constructing every entry keeps the example honest. """ @@ -279,7 +297,14 @@ def test_every_example_coordinator_constructs(self, sdk_config): coordinators=coordinators_example, queue_to_coordinator=json.dumps(queue_to_coordinator), ) - manager = CoordinatorManager.from_config() + # The deployment registers the Dag bundles the example names. + artifact_bundles = [ + {"name": name, "classpath": "airflow.dag_processing.bundles.local.LocalDagBundle", "kwargs": {}} + for spec in specs.values() + if (name := spec.get("kwargs", {}).get("task_handler_bundle_name")) + ] + with conf_vars({("dag_processor", "dag_bundle_config_list"): json.dumps(artifact_bundles)}): + manager = CoordinatorManager.from_config() assert set(manager._coordinator_specs) == set(specs) for queue, key in queue_to_coordinator.items(): From ace4b2eeaa9855b389ca57741a9e94482a5601a6 Mon Sep 17 00:00:00 2001 From: LIU ZHE YOU Date: Wed, 30 Sep 2026 08:57:00 +0000 Subject: [PATCH 02/25] Serve lang-SDK harness artifacts from Dag bundles The compose e2e and the k8s lang-SDK test configured coordinators with the explicit artifact roots, which no longer exist. The Java e2e needs three bundles because one bundle is one classpath, and its three coordinators each need their own dependencies. Artifacts stay mounted on the workers only, because only the workers run them; the other components need just the bundle registration for the name to resolve. The k8s init container keeps staging from S3 because the S3 download drops the Go binary's execute bit, which the coordinator requires. --- .../tests/airflow_e2e_tests/conftest.py | 56 ++++++++++++++----- kubernetes-tests/lang_sdk/README.md | 14 ++--- kubernetes-tests/lang_sdk/config/values.yaml | 14 ++++- .../pod_templates/lang_sdk_golang.yaml | 3 +- .../lang_sdk/pod_templates/lang_sdk_java.yaml | 3 +- kubernetes-tests/lang_sdk/stage_artifacts.py | 8 +-- 6 files changed, 70 insertions(+), 28 deletions(-) diff --git a/airflow-e2e-tests/tests/airflow_e2e_tests/conftest.py b/airflow-e2e-tests/tests/airflow_e2e_tests/conftest.py index 39ff3fd0a64b8..db08b27b06397 100644 --- a/airflow-e2e-tests/tests/airflow_e2e_tests/conftest.py +++ b/airflow-e2e-tests/tests/airflow_e2e_tests/conftest.py @@ -293,6 +293,22 @@ def _setup_xcom_object_storage_integration(dot_env_file, tmp_dir): ] +def _dag_bundle_config_with_artifacts(artifact_bundles: dict[str, str]) -> str: + """Return a ``dag_bundle_config_list`` of the Dags folder plus a ``LocalDagBundle`` per artifact path. + + Registration is what makes a coordinator's ``task_handler_bundle_name`` resolvable on the + worker. The artifact directories are mounted only there; the Dag processor finds no files + in them. + """ + local_bundle = "airflow.dag_processing.bundles.local.LocalDagBundle" + bundles = [{"name": "dags-folder", "classpath": local_bundle, "kwargs": {}}] + bundles.extend( + {"name": name, "classpath": local_bundle, "kwargs": {"path": path}} + for name, path in artifact_bundles.items() + ) + return json.dumps(bundles) + + def _run_java_sdk_gradle(workdir, *gradle_argv, capture_output=False, native=False): """Run the Java SDK Gradle wrapper natively or inside the pinned JDK container. @@ -416,7 +432,7 @@ def _setup_java_sdk_integration(dot_env_file, tmp_dir): copyfile(JAVA_DOCKERFILE_PATH, tmp_dir / "Dockerfile.java") # Copy each bundle's JARs into its own directory; the compose bind-mounts - # expose them to the worker, and each JavaCoordinator globs its own dir. + # expose them to the worker, where each is registered as its own Dag bundle. copytree(JAVA_SDK_EXAMPLE_LIBS_PATH, tmp_dir / "java-jars") copytree(SCALA_SPARK_EXAMPLE_LIBS_PATH, tmp_dir / "scala-jars") copytree(JAVA_TEST_BUNDLE_LIBS_PATH, tmp_dir / "java-test-jars") @@ -456,20 +472,27 @@ def _setup_java_sdk_integration(dot_env_file, tmp_dir): ) # One JavaCoordinator per queue on the same worker image, each serving its - # own bundle. The scala-jdk entry pins main_class (Spark's large classpath - # makes Main-Class discovery ambiguous) and carries Spark's Java 17 module - # openings, a small driver heap, and a longer startup timeout for its large - # dependency classpath. + # own artifact bundle (one bundle is one classpath). The scala-jdk entry pins + # main_class (Spark's large classpath makes Main-Class discovery ambiguous) + # and carries Spark's Java 17 module openings, a small driver heap, and a + # longer startup timeout for its large dependency classpath. + dag_bundle_config = _dag_bundle_config_with_artifacts( + { + "java-task-handlers": "/opt/airflow/java-jars", + "scala-task-handlers": "/opt/airflow/scala-jars", + "java-test-task-handlers": "/opt/airflow/java-test-jars", + } + ) coordinator_config = json.dumps( { "java-jdk": { "classpath": "airflow.sdk.coordinators.java.JavaCoordinator", - "kwargs": {"jars_root": ["/opt/airflow/java-jars"]}, + "kwargs": {"task_handler_bundle_name": "java-task-handlers"}, }, "scala-jdk": { "classpath": "airflow.sdk.coordinators.java.JavaCoordinator", "kwargs": { - "jars_root": ["/opt/airflow/scala-jars"], + "task_handler_bundle_name": "scala-task-handlers", "main_class": "org.apache.airflow.example.ScalaSparkBundleBuilder", "jvm_args": ["-Xmx512m", *_SPARK_JAVA_MODULE_OPTIONS], "task_startup_timeout": 60.0, @@ -477,7 +500,7 @@ def _setup_java_sdk_integration(dot_env_file, tmp_dir): }, "java-test-jdk": { "classpath": "airflow.sdk.coordinators.java.JavaCoordinator", - "kwargs": {"jars_root": ["/opt/airflow/java-test-jars"]}, + "kwargs": {"task_handler_bundle_name": "java-test-task-handlers"}, }, } ) @@ -502,6 +525,7 @@ def _setup_java_sdk_integration(dot_env_file, tmp_dir): dot_env_file.write_text( f"AIRFLOW_UID={os.getuid()}\n" # Single-quote the JSON values so Docker Compose reads them literally. + f"AIRFLOW__DAG_PROCESSOR__DAG_BUNDLE_CONFIG_LIST='{dag_bundle_config}'\n" f"AIRFLOW__SDK__COORDINATORS='{coordinator_config}'\n" f"AIRFLOW__SDK__QUEUE_TO_COORDINATOR='{queue_to_coordinator}'\n" f"AIRFLOW_CONN_TEST_HTTP='{test_http_conn}'\n" @@ -598,9 +622,9 @@ def _setup_go_sdk_integration(dot_env_file, tmp_dir): """Set up the go_sdk E2E test mode. Compiles the Go SDK example bundle into a self-contained executable bundle - via the ``airflow-go-pack`` tooling, drops it into the directory the - ``ExecutableCoordinator`` scans, copies the Python stub Dag, and writes the - coordinator configuration. + via the ``airflow-go-pack`` tooling, drops it into the directory registered + as the ``go-task-handlers`` Dag bundle, copies the Python stub Dag, and + writes the coordinator configuration. The packed bundle is a statically linked native executable (built with ``CGO_ENABLED=0``), so the stock Airflow worker image can exec it directly @@ -624,13 +648,14 @@ def _setup_go_sdk_integration(dot_env_file, tmp_dir): copyfile(GO_SDK_DAGS_PATH / "go_examples.py", tmp_dir / "dags" / "go_examples.py") # Coordinator registry: maps the logical name "go-sdk" to ExecutableCoordinator, - # which scans executables_root for the packed bundle by dag_id. + # which scans the go-task-handlers Dag bundle for the packed bundle by dag_id. # Queue mapping: routes tasks on the "golang" queue to "go-sdk". + dag_bundle_config = _dag_bundle_config_with_artifacts({"go-task-handlers": "/opt/airflow/go-bundles"}) coordinator_config = json.dumps( { "go-sdk": { "classpath": "airflow.sdk.coordinators.executable.ExecutableCoordinator", - "kwargs": {"executables_root": ["/opt/airflow/go-bundles"]}, + "kwargs": {"task_handler_bundle_name": "go-task-handlers"}, } } ) @@ -639,6 +664,7 @@ def _setup_go_sdk_integration(dot_env_file, tmp_dir): dot_env_file.write_text( f"AIRFLOW_UID={os.getuid()}\n" # Single-quote the JSON values so Docker Compose reads them literally. + f"AIRFLOW__DAG_PROCESSOR__DAG_BUNDLE_CONFIG_LIST='{dag_bundle_config}'\n" f"AIRFLOW__SDK__COORDINATORS='{coordinator_config}'\n" f"AIRFLOW__SDK__QUEUE_TO_COORDINATOR='{queue_to_coordinator}'\n" # Connection and variable read by the Go example bundle tasks. @@ -730,12 +756,13 @@ def _setup_ts_sdk_integration(dot_env_file, tmp_dir): for dag_file in ("typescript_example.py", "typescript_taskflow_example.py"): copyfile(TS_SDK_EXAMPLE_PATH / "dags" / dag_file, tmp_dir / "dags" / dag_file) + dag_bundle_config = _dag_bundle_config_with_artifacts({"ts-task-handlers": "/opt/airflow/ts-bundles"}) coordinator_config = json.dumps( { "ts": { "classpath": "airflow.sdk.coordinators.node.NodeCoordinator", "kwargs": { - "bundles_root": ["/opt/airflow/ts-bundles"], + "task_handler_bundle_name": "ts-task-handlers", "node_executable": "/opt/nodejs/node", }, } @@ -747,6 +774,7 @@ def _setup_ts_sdk_integration(dot_env_file, tmp_dir): f"AIRFLOW_UID={os.getuid()}\n" f"NODE_IMAGE={NODE_IMAGE}\n" # single-quoted so Docker Compose reads the JSON literally + f"AIRFLOW__DAG_PROCESSOR__DAG_BUNDLE_CONFIG_LIST='{dag_bundle_config}'\n" f"AIRFLOW__SDK__COORDINATORS='{coordinator_config}'\n" f"AIRFLOW__SDK__QUEUE_TO_COORDINATOR='{queue_to_coordinator}'\n" "AIRFLOW_CONN_TYPESCRIPT_EXAMPLE_HTTP=http://user:pass@example.com/\n" diff --git a/kubernetes-tests/lang_sdk/README.md b/kubernetes-tests/lang_sdk/README.md index 93583ecbd8b44..f19ae247529eb 100644 --- a/kubernetes-tests/lang_sdk/README.md +++ b/kubernetes-tests/lang_sdk/README.md @@ -36,16 +36,16 @@ End-to-end test that one Dag mixing **Python + Go + Java** tasks runs to success worker pod (from that pod template): initContainer stage_artifacts.py ── S3DagBundle.initialize() ──► pulls go-artifacts / java-artifacts bucket into the shared - emptyDir = executables_root / jars_root + emptyDir = go-task-handlers / java-task-handlers LocalDagBundle base container supervisor → coordinator forks the Go binary / Java jar ``` -Key point: in coordinator mode the worker pod does **not** run the Python task-runner, -so the artifact is not downloaded by the normal Dag-bundle path. The init container runs -`stage_artifacts.py`, which reuses the **DagBundle interface** +Key point: each coordinator's `task_handler_bundle_name` names a `LocalDagBundle` over the +shared `emptyDir`, registered in `dagProcessor.dagBundleConfigList`. The init container fills it +by running `stage_artifacts.py`, which reuses the **DagBundle interface** (`DagBundlesManager().get_bundle(name).initialize()` — the download half of -`task_runner.parse`) to pull the artifact from its S3 bucket into the path the -coordinator scans. +`task_runner.parse`) to pull the artifact from its S3 bucket, then restores the Go binary's +execute bit, which the S3 download drops and the coordinator requires. ## Components @@ -58,7 +58,7 @@ coordinator scans. | `pod_templates/lang_sdk_golang.yaml` | `golang` queue worker pod: prod image + go-artifacts init container. | | `pod_templates/lang_sdk_java.yaml` | `java` queue worker pod: JVM image + java-artifacts init container. | | `manifests/localstack.yaml` | In-cluster S3 (localstack). | -| `config/values.yaml` | Helm overrides: KubernetesExecutor, coordinators (+extra.pod_template_file), queue routing, stub-Dag S3 bundle, AWS conn, scheduler pod-template mount. | +| `config/values.yaml` | Helm overrides: KubernetesExecutor, coordinators (+extra.pod_template_file), queue routing, stub-Dag S3 bundle, artifact Dag bundles, AWS conn, scheduler pod-template mount. | The Go binary, Java jar, and stub Dag share one object store (localstack) but live in **separate buckets** (`go-artifacts`, `java-artifacts`, `dags`). diff --git a/kubernetes-tests/lang_sdk/config/values.yaml b/kubernetes-tests/lang_sdk/config/values.yaml index a4b5b4004b403..6f4d9671ad6a5 100644 --- a/kubernetes-tests/lang_sdk/config/values.yaml +++ b/kubernetes-tests/lang_sdk/config/values.yaml @@ -50,6 +50,10 @@ scheduler: # Configure it here (dagProcessor.dagBundleConfigList) rather than overriding # config.dag_processor.dag_bundle_config_list directly -- the chart renders the config from this and # it is the recommended, drift-safe path. +# +# go-task-handlers / java-task-handlers are the coordinators' task_handler_bundle_name bundles: the +# worker pod's init container stages each artifact bucket into these paths (stage_artifacts.py). +# They exist only in the worker pods, so the dag-processor finds no files in them. dagProcessor: dagBundleConfigList: - name: dags-folder @@ -60,6 +64,14 @@ dagProcessor: kwargs: bucket_name: "dags" aws_conn_id: "aws_localstack" + - name: go-task-handlers + classpath: "airflow.dag_processing.bundles.local.LocalDagBundle" + kwargs: + path: "/opt/airflow/artifacts/go-artifacts" + - name: java-task-handlers + classpath: "airflow.dag_processing.bundles.local.LocalDagBundle" + kwargs: + path: "/opt/airflow/artifacts/java-artifacts" config: # Route the golang/java queues to their coordinators, each carrying an @@ -70,5 +82,5 @@ config: # Single-line JSON: airflow.cfg is an ini file, so a value with embedded # newlines would break the rendered configmap. # yamllint disable-line rule:line-length - coordinators: '{"go-sdk": {"classpath": "airflow.sdk.coordinators.executable.ExecutableCoordinator", "kwargs": {"executables_root": ["/opt/airflow/artifacts/go-artifacts"]}, "extra": {"pod_template_file": "/opt/airflow/lang_sdk/pod_templates/lang_sdk_golang.yaml"}}, "java-sdk": {"classpath": "airflow.sdk.coordinators.java.JavaCoordinator", "kwargs": {"jars_root": ["/opt/airflow/artifacts/java-artifacts"]}, "extra": {"pod_template_file": "/opt/airflow/lang_sdk/pod_templates/lang_sdk_java.yaml", "worker_container_repository": "lang-sdk-java-worker", "worker_container_tag": "latest"}}}' + coordinators: '{"go-sdk": {"classpath": "airflow.sdk.coordinators.executable.ExecutableCoordinator", "kwargs": {"task_handler_bundle_name": "go-task-handlers"}, "extra": {"pod_template_file": "/opt/airflow/lang_sdk/pod_templates/lang_sdk_golang.yaml"}}, "java-sdk": {"classpath": "airflow.sdk.coordinators.java.JavaCoordinator", "kwargs": {"task_handler_bundle_name": "java-task-handlers"}, "extra": {"pod_template_file": "/opt/airflow/lang_sdk/pod_templates/lang_sdk_java.yaml", "worker_container_repository": "lang-sdk-java-worker", "worker_container_tag": "latest"}}}' queue_to_coordinator: '{"golang": "go-sdk", "java": "java-sdk"}' diff --git a/kubernetes-tests/lang_sdk/pod_templates/lang_sdk_golang.yaml b/kubernetes-tests/lang_sdk/pod_templates/lang_sdk_golang.yaml index e3e700d700012..00d6b990690ae 100644 --- a/kubernetes-tests/lang_sdk/pod_templates/lang_sdk_golang.yaml +++ b/kubernetes-tests/lang_sdk/pod_templates/lang_sdk_golang.yaml @@ -19,7 +19,8 @@ # # An init container stages the packed Go bundle from the localstack ``go-artifacts`` # bucket into a shared emptyDir at /opt/airflow/artifacts via the DagBundle -# interface (stage_artifacts.py). That path is the coordinator's executables_root. +# interface (stage_artifacts.py). That path is the go-task-handlers Dag bundle the +# coordinator's task_handler_bundle_name names. # The base image is the plain prod image: a Go bundle is a self-contained static # binary, so no extra runtime is needed. Image placeholders are rendered by the # ``breeze k8s setup-lang-sdk-test`` command. diff --git a/kubernetes-tests/lang_sdk/pod_templates/lang_sdk_java.yaml b/kubernetes-tests/lang_sdk/pod_templates/lang_sdk_java.yaml index eaccd63f4d2ad..ad4456610fb73 100644 --- a/kubernetes-tests/lang_sdk/pod_templates/lang_sdk_java.yaml +++ b/kubernetes-tests/lang_sdk/pod_templates/lang_sdk_java.yaml @@ -19,7 +19,8 @@ # # An init container stages the Java jar from the localstack ``java-artifacts`` # bucket into a shared emptyDir at /opt/airflow/artifacts via the DagBundle -# interface (stage_artifacts.py). That path is the coordinator's jars_root. The +# interface (stage_artifacts.py). That path is the java-task-handlers Dag bundle the +# coordinator's task_handler_bundle_name names. The # base image is the prod image plus a headless JRE (built from Dockerfile.java) # so the JavaCoordinator can exec ``java`` -- distinct from the Go queue's plain # prod image. Image placeholders are rendered by ``breeze k8s setup-lang-sdk-test``. diff --git a/kubernetes-tests/lang_sdk/stage_artifacts.py b/kubernetes-tests/lang_sdk/stage_artifacts.py index 3da9ac6fb5a72..45a9d69882b03 100644 --- a/kubernetes-tests/lang_sdk/stage_artifacts.py +++ b/kubernetes-tests/lang_sdk/stage_artifacts.py @@ -23,10 +23,10 @@ parse), so the Go binary / Java jar stored in an S3 (localstack) bucket is pulled into ``bundle.path`` = ``{dag_bundle_storage_path}/{name}``. -That path is the ``emptyDir`` shared with the worker container and is exactly the -``executables_root`` / ``jars_root`` the coordinator scans. Coordinator mode never -runs the Python task-runner in the worker pod, so without this step the artifact -would never reach the pod. +That path is the ``emptyDir`` shared with the worker container, where it is registered +as the ``LocalDagBundle`` the coordinator's ``task_handler_bundle_name`` names. The +coordinator does not name the S3 bundle directly because its download drops the Go +binary's execute bit, which the coordinator requires; Java is staged the same way. Configuration is read from the environment (set by the pod template): From 4bfc18153e83066b031a9150634f21825bf9ccc0 Mon Sep 17 00:00:00 2001 From: LIU ZHE YOU Date: Wed, 30 Sep 2026 09:04:57 +0000 Subject: [PATCH 03/25] Document artifact Dag bundles for language SDK coordinators The Go, Java and TypeScript guides, the bundle specs, the READMEs and the contributor guide still pointed users at the removed jars_root, executables_root and bundles_root kwargs. A Dag bundle now decides what a coordinator sees, so two of its limits reach users. For Java, every JAR in the bundle goes on one classpath, so conflicting dependency versions cannot share a bundle, and a second Main-Class in it makes the entry point ambiguous. For Go, the coordinator skips files without the execute bit, so an object-store Dag bundle that drops it leaves the coordinator with no executable to find. --- .agents/skills/airflow-java-sdk/SKILL.md | 11 ++- .../language-sdks/go.rst | 82 +++++++++-------- .../language-sdks/java.rst | 92 ++++++++++--------- .../language-sdks/typescript.rst | 79 ++++++++-------- contributing-docs/30_new_language_sdk.rst | 8 +- go-sdk/README.md | 35 ++++--- go-sdk/example/bundle/Justfile | 5 +- java-sdk/README.md | 19 +++- .../airflow/sdk/plugin/AirflowSdkPlugin.kt | 4 +- java-sdk/scala_spark_example/README.md | 6 +- task-sdk/docs/executable-bundle-spec.rst | 27 +++--- task-sdk/docs/ts-bundle-spec.rst | 26 +++--- ts-sdk/README.md | 17 +++- ts-sdk/example/README.md | 14 ++- 14 files changed, 246 insertions(+), 179 deletions(-) diff --git a/.agents/skills/airflow-java-sdk/SKILL.md b/.agents/skills/airflow-java-sdk/SKILL.md index 8edb742679535..9921832eafee5 100644 --- a/.agents/skills/airflow-java-sdk/SKILL.md +++ b/.agents/skills/airflow-java-sdk/SKILL.md @@ -50,8 +50,9 @@ subclasses must only import from `org.apache.airflow.sdk`; any import of ## Bundle composition and coordinator discovery -A **bundle** is a directory of JAR files (typically `build/bundle/`) placed on the coordinator's -`jars_root`. The coordinator scans the directory at task-dispatch time to find: +A **bundle** is a directory of JAR files (typically `build/bundle/`) placed in the Dag bundle named +by the coordinator's `task_handler_bundle_name` (the task's own Dag bundle when unset). The +coordinator scans that Dag bundle at task-dispatch time to find: 1. **`Main-Class`** (standard JAR manifest attribute) — the fully-qualified class name of the entry point that the coordinator invokes with `java -classpath … --comm … --logs …`. @@ -65,7 +66,7 @@ A **bundle** is a directory of JAR files (typically `build/bundle/`) placed on t `runtimeClasspath` and copies it into the shadow JAR manifest. In thin-JAR mode (`fatJar = false`), the value stays in the `airflow-sdk` JAR deployed alongside the bundle JAR. -The Python coordinator (`JavaCoordinator`) scans every JAR under `jars_root` with +The Python coordinator (`JavaCoordinator`) scans every JAR in that Dag bundle with `_JarInfo.find()`, reads `META-INF/MANIFEST.MF` out of each ZIP, and collects `Main-Class` and `Airflow-Supervisor-Schema-Version` from whichever JARs carry them. The resolved schema version is then passed as the `schema_version` return value from `_build_execute_task_command`, which @@ -74,7 +75,7 @@ the base `SubprocessCoordinator` uses to negotiate the supervisor wire protocol. If `main_class` is set explicitly on the `JavaCoordinator` instance (via `[sdk] coordinators` kwargs), the scan uses it as a filter; otherwise the first JAR with a `Main-Class` attribute wins. Either way, `Airflow-Supervisor-Schema-Version` must be present in at least one JAR in -`jars_root` or startup fails. +the Dag bundle or startup fails. Every JAR in the Dag bundle goes on one classpath. --- @@ -117,7 +118,7 @@ E2E_TEST_MODE=java_sdk uv run --project airflow-e2e-tests pytest \ `coordinator.py` extends `SubprocessCoordinator`. The only method subclasses must implement is `_build_execute_task_command`, which returns `(argv, schema_version)`. Look at the existing -implementation for how `jars_root`, `java_executable`, `jvm_args`, and `main_class` are +implementation for how the scanned Dag bundle, `java_executable`, `jvm_args`, and `main_class` are assembled into the command. Do not reach into the JVM process from Python beyond what this method provides. diff --git a/airflow-core/docs/authoring-and-scheduling/language-sdks/go.rst b/airflow-core/docs/authoring-and-scheduling/language-sdks/go.rst index bd2a3e4ac761a..b0599035f9601 100644 --- a/airflow-core/docs/authoring-and-scheduling/language-sdks/go.rst +++ b/airflow-core/docs/authoring-and-scheduling/language-sdks/go.rst @@ -48,7 +48,7 @@ Prerequisites * Go 1.24 or later to build and pack bundles. This is a build-time requirement only; the worker that runs a packed bundle needs no Go toolchain, because the bundle is a self-contained native executable. -* The packed bundle must be accessible from the Airflow worker, under a directory the coordinator scans. +* The packed bundle must be accessible from the Airflow worker, in the Dag bundle the coordinator scans. * The ``apache-airflow-task-sdk`` package (installed with Airflow) provides the coordinator; no additional Python packages are needed. @@ -178,33 +178,44 @@ to pass on with ``bundle.Register(reports.Handlers()...)``. Coordinator configuration ~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Register the coordinator and route the queue to it under ``[sdk]`` in ``airflow.cfg`` (or the equivalent -``AIRFLOW__SDK__*`` environment variables): +Register a Dag bundle for the packed bundles, register the coordinator, and route the queue to it in +``airflow.cfg`` (or the equivalent ``AIRFLOW__*`` environment variables): .. code-block:: ini + [dag_processor] + dag_bundle_config_list = [ + {"name": "dags-folder", "classpath": "airflow.dag_processing.bundles.local.LocalDagBundle", "kwargs": {}}, + { + "name": "go-task-handlers", + "classpath": "airflow.dag_processing.bundles.local.LocalDagBundle", + "kwargs": {"path": "/opt/airflow/go-task-handlers"} + } + ] + [sdk] coordinators = { "go": { "classpath": "airflow.sdk.coordinators.executable.ExecutableCoordinator", - "kwargs": {"executables_root": ["~/airflow/executable-bundles"]} + "kwargs": {"task_handler_bundle_name": "go-task-handlers"} } } queue_to_coordinator = {"golang": "go"} -``executables_root`` is one or more directories the coordinator scans for bundles; ``queue_to_coordinator`` -routes stub tasks with ``queue="golang"`` to this Go coordinator. See :ref:`go-sdk/coordinator-config` for -the full list of accepted ``kwargs``. +``task_handler_bundle_name`` names the Dag bundle the coordinator scans for packed bundles; +``queue_to_coordinator`` routes stub tasks with ``queue="golang"`` to this Go coordinator. See +:ref:`go-sdk/coordinator-config` for the full list of accepted ``kwargs`` and how bundles are located. There is no separate Go worker to run: the Airflow worker forks the bundle binary once per task instance. .. note:: - The coordinator is part of the Airflow worker, so the ``[sdk]`` config (and the bundle files in - ``executables_root``) only need to be present wherever tasks actually execute. With ``CeleryExecutor``, - setting it on the Celery workers is sufficient. With ``LocalExecutor``, tasks run inside the scheduler - process, so it must be set where the scheduler can read it. The API server and Dag processor do not need - it. + The coordinator is part of the Airflow worker, so the ``[sdk]`` config and the packed bundle files only + need to be present wherever tasks actually execute. With ``CeleryExecutor``, setting it on the Celery + workers is sufficient. With ``LocalExecutor``, tasks run inside the scheduler process, so it must be set + where the scheduler can read it. The API server and Dag processor do not need it. The worker resolves + ``task_handler_bundle_name`` through ``[dag_processor] dag_bundle_config_list``, so the workers need that + setting too. Writing tasks ------------- @@ -444,12 +455,12 @@ Build and pack in one step; any flags after ``--`` are forwarded verbatim to ``g go tool airflow-go-pack ./example/bundle -- -trimpath -tags=prod -Use ``--output `` to write the packed bundle straight into a directory the coordinator scans -(``executables_root``): +Use ``--output `` to write the packed bundle straight into the directory of the Dag bundle the +coordinator scans (see `Deploying`_): .. code-block:: bash - go tool airflow-go-pack --output ~/airflow/executable-bundles/sample-dag-bundle ./example/bundle + go tool airflow-go-pack --output /opt/airflow/go-task-handlers/sample-dag-bundle ./example/bundle Cross-platform builds ~~~~~~~~~~~~~~~~~~~~~~~ @@ -461,7 +472,7 @@ machine (for example, deploying to a Linux host from an Apple-silicon ``darwin/a .. code-block:: bash go tool airflow-go-pack --goos linux --goarch amd64 \ - --output ~/airflow/executable-bundles/sample-dag-bundle \ + --output /opt/airflow/go-task-handlers/sample-dag-bundle \ ./example/bundle Alternatively, pack a pre-built binary with ``--executable`` / ``--source``. The packer normally execs the @@ -485,12 +496,16 @@ with ``--airflow-metadata``: Deploying ~~~~~~~~~ -Copy or mount the packed bundle into a directory listed in the coordinator's ``executables_root``. The -:class:`~airflow.sdk.coordinators.executable.ExecutableCoordinator` scans those directories recursively, +Copy or mount the packed bundle into the Dag bundle named by the coordinator's ``task_handler_bundle_name``. +The :class:`~airflow.sdk.coordinators.executable.ExecutableCoordinator` scans that Dag bundle recursively, matches the incoming ``dag_id`` against each bundle's manifest, verifies the bundle's integrity hash, and launches the matching bundle. Bundles are identified by the trailer magic, not by filename (no extension on Linux/macOS, ``.exe`` on Windows), so the file name on the worker is irrelevant. +Only files with the executable bit set are considered, so the packed bundles need a Dag bundle that keeps +it. A ``LocalDagBundle`` over a directory you manage does; object-store Dag bundles such as ``S3DagBundle`` +do not. + .. _go-sdk/coordinator-config: :class:`~airflow.sdk.coordinators.executable.ExecutableCoordinator` configuration @@ -506,16 +521,11 @@ All ``kwargs`` in the ``coordinators`` config entry are passed to the * - Parameter - Default - Description - * - ``executables_root`` - - *(optional)* - - One or more directories scanned recursively for executable bundles. Accepts a string, - a path, or a list of strings/paths. When omitted, bundles are located through a Dag - bundle instead (see the note below). Explicitly setting this option to ``null`` or - an empty list is invalid. - * - ``dag_bundle_name`` - - *(auto: task's own bundle)* - - Name of a configured Dag bundle to load executable bundles from. Mutually exclusive - with ``executables_root``. + * - ``task_handler_bundle_name`` + - *(task's own Dag bundle)* + - Name of the Dag bundle scanned recursively for executable bundles. It must be registered in + ``[dag_processor] dag_bundle_config_list``. It is checked when the ``[sdk]`` configuration is + loaded, so a typo fails there rather than on the first task. * - ``task_startup_timeout`` - ``10.0`` - Seconds to wait for the bundle subprocess to connect after launch. Increase this if your @@ -523,15 +533,15 @@ All ``kwargs`` in the ``coordinators`` config entry are passed to the .. note:: - **Locating bundles.** ``executables_root`` and ``dag_bundle_name`` are mutually exclusive, - and both are optional: + **Locating bundles.** Packed bundles are read from a Dag bundle, so they are delivered, refreshed and + versioned by the same machinery as your Dags. - * Set ``executables_root`` to scan explicit filesystem directories you manage yourself. - * Set ``dag_bundle_name`` to load bundles from a configured Dag bundle, so they are delivered - and versioned through the same bundle machinery as your Dags. The task uses the version that - bundle is on when it starts, pinned for the whole task. - * Leave both unset (the default) to load bundles from the **task's own** Dag bundle, pinned - to the version the run was created with. + * The expected layout is a separate Dag bundle for the packed bundles, named by + ``task_handler_bundle_name``, rather than the Dag bundle that holds your ``.py`` files. The task + uses the version that Dag bundle is on when it starts, pinned for the whole task. + * If ``task_handler_bundle_name`` is unset, packed bundles are read from the **task's own** Dag + bundle, pinned to the version the run was created with. + * The Dag bundle must keep the executable bit (see `Deploying`_). .. _go-sdk/limitations: diff --git a/airflow-core/docs/authoring-and-scheduling/language-sdks/java.rst b/airflow-core/docs/authoring-and-scheduling/language-sdks/java.rst index 3f69772e4d92d..5f875617e57e9 100644 --- a/airflow-core/docs/authoring-and-scheduling/language-sdks/java.rst +++ b/airflow-core/docs/authoring-and-scheduling/language-sdks/java.rst @@ -56,7 +56,8 @@ deployment process are the same. See :ref:`java-sdk/interface-api` for the inter The Python Dag source and the Java Gradle project are independent. They do not need to be in the same repository or have any particular relative filesystem layout. The Dag follows the deployment's normal Dag -delivery process; only the compiled Java bundle is deployed from the Gradle project to ``jars_root``. +delivery process; only the compiled Java bundle is deployed from the Gradle project, into a separate Dag +bundle that holds the JARs. Define the Python Dag ~~~~~~~~~~~~~~~~~~~~~ @@ -245,29 +246,42 @@ Deploy ``sales_pipeline.py`` separately through the deployment's normal Dag deli that process might sync it to ``${AIRFLOW_HOME}/dags/`` or package it in a Dag bundle; neither location is inside or relative to ``sales-pipeline-java/``. -Configure Airflow so the coordinator scans the parent JAR directory recursively and routes the ``java`` queue -to it. Add the following ``[sdk]`` section to the file selected by ``AIRFLOW_CONFIG`` (by default, -``${AIRFLOW_HOME}/airflow.cfg``), or set the equivalent ``AIRFLOW__SDK__*`` environment variables: +Configure Airflow to register the JAR directory as a Dag bundle, point the coordinator at it, and route the +``java`` queue to the coordinator. Add the following sections to the file selected by ``AIRFLOW_CONFIG`` (by +default, ``${AIRFLOW_HOME}/airflow.cfg``), or set the equivalent ``AIRFLOW__*`` environment variables: .. code-block:: ini + [dag_processor] + dag_bundle_config_list = [ + {"name": "dags-folder", "classpath": "airflow.dag_processing.bundles.local.LocalDagBundle", "kwargs": {}}, + { + "name": "java-task-handlers", + "classpath": "airflow.dag_processing.bundles.local.LocalDagBundle", + "kwargs": {"path": "/opt/airflow/jars/sales-pipeline"} + } + ] + [sdk] coordinators = { "java": { "classpath": "airflow.sdk.coordinators.java.JavaCoordinator", - "kwargs": {"jars_root": ["/opt/airflow/jars"]} + "kwargs": {"task_handler_bundle_name": "java-task-handlers"} } } queue_to_coordinator = {"java": "java"} ``java`` is a user-chosen coordinator name, not a reserved value. The value assigned to the queue in -``queue_to_coordinator`` must match a key in ``coordinators``. +``queue_to_coordinator`` must match a key in ``coordinators``, and ``task_handler_bundle_name`` must match a +Dag bundle name in ``dag_bundle_config_list``. See :ref:`java-sdk/coordinator-config` for how JARs are +located. Restart the affected Airflow components after changing this configuration. The coordinator config and JARs must be available wherever tasks execute. With ``CeleryExecutor``, that means the Celery workers; with -``LocalExecutor``, tasks run in subprocesses on the scheduler's host. The API server and Dag processor do not -need the JARs, while the Dag processor must receive ``sales_pipeline.py`` through the separate Dag delivery -process. +``LocalExecutor``, tasks run in subprocesses on the scheduler's host. The worker resolves +``task_handler_bundle_name`` through ``[dag_processor] dag_bundle_config_list``, so the workers need that +setting too. The API server and Dag processor do not need the JARs, while the Dag processor must receive +``sales_pipeline.py`` through the separate Dag delivery process. After Airflow has parsed the Dag, trigger it from the UI or command line: @@ -726,7 +740,7 @@ configuration): "java-jdk17": { "classpath": "airflow.sdk.coordinators.java.JavaCoordinator", "kwargs": { - "jars_root": ["/opt/airflow/jars"], + "task_handler_bundle_name": "java-task-handlers", "jvm_args": ["-Djava.util.logging.config.file=/opt/airflow/logging.properties"] } } @@ -851,9 +865,10 @@ Then run: ./gradlew bundle -The ``build/bundle/`` directory contains all required JAR(s). Copy or mount it into the directory pointed to -by ``jars_root`` in the coordinator configuration. :class:`~airflow.sdk.coordinators.java.JavaCoordinator` -scans ``jars_root`` recursively and builds the classpath automatically. +The ``build/bundle/`` directory contains all required JAR(s). Copy or mount it into the Dag bundle named by +``task_handler_bundle_name`` in the coordinator configuration. +:class:`~airflow.sdk.coordinators.java.JavaCoordinator` scans that Dag bundle recursively and builds the +classpath automatically. .. note:: @@ -865,8 +880,7 @@ scans ``jars_root`` recursively and builds the classpath automatically. The plugin generates a fat JAR with the `Shadow `__ plugin by default. This is generally a good idea since you only deploy one JAR file to avoid dependency issues between projects. If this does not suit you, set ``fatJar = false`` in ``airflowBundle`` to produce thin JARs instead. The rest of the - process stays the same, but you will need to put all dependency JARs somewhere Airflow can find with - ``jars_root``. + process stays the same, but you will need to put all dependency JARs in the same Dag bundle. .. _java-sdk/build/maven: @@ -956,8 +970,8 @@ Then run: mvn package -The fat JAR is written to ``target/-.jar``. Copy it to the directory configured as -``jars_root`` in your coordinator. +The fat JAR is written to ``target/-.jar``. Copy it into the Dag bundle named by +``task_handler_bundle_name`` in your coordinator. **Option 2: thin JAR with separate dependencies** @@ -1017,8 +1031,8 @@ Then run: mvn package -``target/bundle/`` will contain the thin JAR and all runtime dependency JARs. Point ``jars_root`` at -this directory. +``target/bundle/`` will contain the thin JAR and all runtime dependency JARs. Copy or mount this +directory into the Dag bundle named by ``task_handler_bundle_name``. .. note:: @@ -1044,16 +1058,11 @@ All ``kwargs`` in the ``coordinators`` config entry are passed to the * - Parameter - Default - Description - * - ``jars_root`` - - *(optional)* - - One or more directories scanned recursively for ``.jar`` files. Accepts a string, - a path, or a list of strings/paths. When omitted, JARs are located through a Dag - bundle instead (see the note below). Explicitly setting this option to ``null`` or - an empty list is invalid. - * - ``dag_bundle_name`` - - *(auto: task's own bundle)* - - Name of a configured Dag bundle to load JARs from. Mutually exclusive with - ``jars_root``. + * - ``task_handler_bundle_name`` + - *(task's own Dag bundle)* + - Name of the Dag bundle scanned recursively for ``.jar`` files. It must be registered in + ``[dag_processor] dag_bundle_config_list``. It is checked when the ``[sdk]`` configuration is + loaded, so a typo fails there rather than on the first task. * - ``java_executable`` - ``"java"`` - Path to the ``java`` binary. Defaults to ``java`` on ``$PATH``. @@ -1062,10 +1071,9 @@ All ``kwargs`` in the ``coordinators`` config entry are passed to the - Extra JVM arguments such as ``["-Xmx1g", "-Dsome.property=value"]``. * - ``main_class`` - *(auto-detect)* - - Explicit entry-point class. If omitted, the coordinator scans for a JAR whose - manifest sets ``Main-Class`` — in ``jars_root`` when set, otherwise across the - resolved Dag bundle. If multiple executable JARs match the result is - non-deterministic; set ``main_class`` explicitly in that case. + - Explicit entry-point class. If omitted, the coordinator scans the Dag bundle for a JAR + whose manifest sets ``Main-Class``. If more than one JAR in that Dag bundle sets it, which + one runs is non-deterministic, so set ``main_class`` explicitly in that case. * - ``task_startup_timeout`` - ``10.0`` - Seconds to wait for the JVM subprocess to connect after launch. Increase this if your @@ -1073,15 +1081,17 @@ All ``kwargs`` in the ``coordinators`` config entry are passed to the .. note:: - **Locating JARs.** ``jars_root`` and ``dag_bundle_name`` are mutually exclusive, and both - are optional: + **Locating JARs.** JARs are read from a Dag bundle, so they are delivered, refreshed and versioned + by the same machinery as your Dags. - * Set ``jars_root`` to scan explicit filesystem directories you manage yourself. - * Set ``dag_bundle_name`` to load JARs from a configured Dag bundle, so they are delivered and - versioned through the same bundle machinery as your Dags. The task uses the version that bundle + * The expected layout is a separate Dag bundle for the JARs, named by ``task_handler_bundle_name``, + rather than the Dag bundle that holds your ``.py`` files. The task uses the version that Dag bundle is on when it starts, pinned for the whole task. - * Leave both unset (the default) to load JARs from the **task's own** Dag bundle, pinned to - the version the run was created with. + * If ``task_handler_bundle_name`` is unset, JARs are read from the **task's own** Dag bundle, pinned + to the version the run was created with. + * Every JAR in the Dag bundle goes on one classpath, so all handlers in it share one set of + dependencies. To isolate conflicting dependency versions, put the handlers in a second Dag bundle + served by a second coordinator on its own queue. .. note:: @@ -1110,7 +1120,7 @@ point ``java_executable`` at it explicitly: "java-jdk17": { "classpath": "airflow.sdk.coordinators.java.JavaCoordinator", "kwargs": { - "jars_root": ["/opt/airflow/jars"], + "task_handler_bundle_name": "java-task-handlers", "java_executable": "/opt/homebrew/opt/openjdk@17/bin/java" } } diff --git a/airflow-core/docs/authoring-and-scheduling/language-sdks/typescript.rst b/airflow-core/docs/authoring-and-scheduling/language-sdks/typescript.rst index 47070c581da15..452a9dca2c68c 100644 --- a/airflow-core/docs/authoring-and-scheduling/language-sdks/typescript.rst +++ b/airflow-core/docs/authoring-and-scheduling/language-sdks/typescript.rst @@ -50,7 +50,7 @@ Prerequisites * Node.js 22 or later must be available on the Airflow worker nodes. * The packed bundle (a single ``bundle.min.mjs`` file, see :ref:`typescript-sdk/build`) must be accessible - from the worker, under a directory the coordinator scans. + from the worker, in the Dag bundle the coordinator scans. * The ``apache-airflow-task-sdk`` package (installed with Airflow) provides the coordinator; no additional Python packages are needed. * In the TypeScript project, install the ``apache-airflow-ts-sdk`` npm package to author task handlers: @@ -209,34 +209,46 @@ check. Coordinator configuration ~~~~~~~~~~~~~~~~~~~~~~~~~ -Register the coordinator and route the queue to it under ``[sdk]`` in ``airflow.cfg`` (or the equivalent -``AIRFLOW__SDK__*`` environment variables): +Register a Dag bundle for the packed bundles, register the coordinator, and route the queue to it in +``airflow.cfg`` (or the equivalent ``AIRFLOW__*`` environment variables): .. code-block:: ini + [dag_processor] + dag_bundle_config_list = [ + {"name": "dags-folder", "classpath": "airflow.dag_processing.bundles.local.LocalDagBundle", "kwargs": {}}, + { + "name": "ts-task-handlers", + "classpath": "airflow.dag_processing.bundles.local.LocalDagBundle", + "kwargs": {"path": "/opt/airflow/ts-bundles"} + } + ] + [sdk] coordinators = { "ts": { "classpath": "airflow.sdk.coordinators.node.NodeCoordinator", - "kwargs": {"bundles_root": ["/opt/airflow/ts-bundles"]} + "kwargs": {"task_handler_bundle_name": "ts-task-handlers"} } } queue_to_coordinator = {"typescript": "ts"} -``bundles_root`` is one or more directories the coordinator scans for bundles; ``queue_to_coordinator`` -routes stub tasks with ``queue="typescript"`` to this coordinator. See -:ref:`typescript-sdk/coordinator-config` for the full list of accepted ``kwargs``. +``task_handler_bundle_name`` names the Dag bundle the coordinator scans for packed bundles; +``queue_to_coordinator`` routes stub tasks with ``queue="typescript"`` to this coordinator. See +:ref:`typescript-sdk/coordinator-config` for the full list of accepted ``kwargs`` and how the bundle is +located. There is no separate Node.js worker to run: the Airflow worker launches the bundle with ``node`` once per task instance. .. note:: - The coordinator runs inside the Airflow worker, so the ``[sdk]`` config (and the packed ``*.min.mjs`` - bundles in ``bundles_root``) only need to be present wherever tasks actually execute. With - ``CeleryExecutor``, setting them on the Celery workers is sufficient. With ``LocalExecutor``, tasks run - inside the scheduler process, so they must be present where the scheduler can read them. The API server - and Dag processor do not need them. + The coordinator runs inside the Airflow worker, so the ``[sdk]`` config and the packed ``*.min.mjs`` + bundles only need to be present wherever tasks actually execute. With ``CeleryExecutor``, setting them on + the Celery workers is sufficient. With ``LocalExecutor``, tasks run inside the scheduler process, so they + must be present where the scheduler can read them. The API server and Dag processor do not need them. The + worker resolves ``task_handler_bundle_name`` through ``[dag_processor] dag_bundle_config_list``, so the + workers need that setting too. .. _typescript-sdk/native-dag: @@ -499,7 +511,7 @@ that only supply utilities or types are not embedded. npx airflow-ts-pack src/main.ts --outdir dist Use ``--outdir `` to choose the output directory (default ``dist``), ``--outfile `` to name the -artifact exactly, which helps when one ``bundles_root`` holds several bundles, and ``--source `` to set +artifact exactly, which helps when one Dag bundle holds several bundles, and ``--source `` to set the source name displayed in the Airflow UI (default: the entry file's basename). ``--outdir`` and ``--outfile`` are mutually exclusive, and an ``--outfile`` name must end in ``.min.mjs`` so the coordinator can find it. @@ -507,12 +519,11 @@ can find it. Deploying ~~~~~~~~~ -Copy or mount the bundle into a directory listed in the coordinator's ``bundles_root``. -:class:`~airflow.sdk.coordinators.node.NodeCoordinator` searches the configured directories in order, -recursively, and launches the first integrity-verified ``*.min.mjs`` bundle whose metadata declares the task -instance's Dag. The artifact's name does not matter beyond that suffix, so one root can hold several bundles -and a Dag is routed to whichever declares it. If multiple bundles declare the same Dag, the first configured -root wins, and within a root the first in sorted path order. +Copy or mount the bundle into the Dag bundle named by the coordinator's ``task_handler_bundle_name``. +:class:`~airflow.sdk.coordinators.node.NodeCoordinator` searches that Dag bundle recursively and launches the +first integrity-verified ``*.min.mjs`` bundle whose metadata declares the task instance's Dag. The artifact's +name does not matter beyond that suffix, so one Dag bundle can hold several bundles and a Dag is routed to +whichever declares it. If multiple bundles declare the same Dag, the first in sorted path order wins. .. _typescript-sdk/coordinator-config: @@ -529,16 +540,11 @@ All ``kwargs`` in the ``coordinators`` config entry are passed to the * - Parameter - Default - Description - * - ``bundles_root`` - - *(optional)* - - One or more directories searched recursively, in order, for an integrity-verified ``*.min.mjs`` - bundle that declares the requested Dag. Accepts a string, a path, or a list of strings/paths. When - omitted, the bundle is located through a Dag bundle instead (see the note below). Explicitly setting - this option to ``null`` or an empty list is invalid. - * - ``dag_bundle_name`` - - *(auto: task's own bundle)* - - Name of a configured Dag bundle to load the ``*.min.mjs`` bundle from. Mutually exclusive with - ``bundles_root``. + * - ``task_handler_bundle_name`` + - *(task's own Dag bundle)* + - Name of the Dag bundle searched recursively for an integrity-verified ``*.min.mjs`` bundle that + declares the requested Dag. It must be registered in ``[dag_processor] dag_bundle_config_list``. It is + checked when the ``[sdk]`` configuration is loaded, so a typo fails there rather than on the first task. * - ``node_executable`` - ``"node"`` - Path to the ``node`` binary. Defaults to ``node`` on ``$PATH``. @@ -549,15 +555,14 @@ All ``kwargs`` in the ``coordinators`` config entry are passed to the .. note:: - **Locating the bundle.** ``bundles_root`` and ``dag_bundle_name`` are mutually exclusive, and both - are optional: + **Locating the bundle.** Packed bundles are read from a Dag bundle, so they are delivered, refreshed and + versioned by the same machinery as your Dags. - * Set ``bundles_root`` to scan explicit filesystem directories you manage yourself. - * Set ``dag_bundle_name`` to load the bundle from a configured Dag bundle, so it is delivered - and versioned through the same bundle machinery as your Dags. The task uses the version that - bundle is on when it starts, pinned for the whole task. - * Leave both unset (the default) to load the bundle from the **task's own** Dag bundle, pinned to the - version the run was created with. + * The expected layout is a separate Dag bundle for the packed bundles, named by + ``task_handler_bundle_name``, rather than the Dag bundle that holds your ``.py`` files. The task uses + the version that Dag bundle is on when it starts, pinned for the whole task. + * If ``task_handler_bundle_name`` is unset, the bundle is read from the **task's own** Dag bundle, pinned + to the version the run was created with. Limitations ----------- diff --git a/contributing-docs/30_new_language_sdk.rst b/contributing-docs/30_new_language_sdk.rst index 34c4cdbb4a27a..86c65efc716ea 100644 --- a/contributing-docs/30_new_language_sdk.rst +++ b/contributing-docs/30_new_language_sdk.rst @@ -121,10 +121,10 @@ The method returns a ``(command, subprocess_schema_version)`` pair: across SDK versions. See `Supervisor Schema`_ below. Call ``self._get_scan_roots()`` to retrieve the artifact directories the base -class has already resolved from the coordinator's configured source — an -explicit filesystem root, a named Dag bundle (``dag_bundle_name``), or the -task's own bundle. Subclasses should scan those roots rather than reading the -configured root directly. +class has already resolved: the Dag bundle named by ``task_handler_bundle_name``, +or the task's own Dag bundle when it is unset, pinned for the whole task. +Subclasses should scan those directories rather than locating artifacts +themselves. Supervisor Schema ~~~~~~~~~~~~~~~~~ diff --git a/go-sdk/README.md b/go-sdk/README.md index a6dedde08049b..5e8dba63ea414 100644 --- a/go-sdk/README.md +++ b/go-sdk/README.md @@ -250,15 +250,15 @@ the full range of task states, and alternate XCom backends without implementing go tool airflow-go-pack ./example/bundle -- -trimpath -tags=prod ``` - Use `--output ` to write the packed bundle straight into a directory the coordinator scans - (`executables_root`), and pass extra `go build` flags after `--`. + Use `--output ` to write the packed bundle straight into the directory of the Dag bundle the + coordinator scans (`task_handler_bundle_name`), and pass extra `go build` flags after `--`. For cross-compiling (e.g. deploy to a Linux host from an Apple-silicon (darwin/arm64) machine), pass `--goos`/`--goarch` and the packer cross-builds for you: ```bash go tool airflow-go-pack --goos linux --goarch amd64 \ - --output ~/airflow/executable-bundles/sample-dag-bundle \ + --output /opt/airflow/go-task-handlers/sample-dag-bundle \ ./example/bundle ``` @@ -280,24 +280,33 @@ the full range of task states, and alternate XCom backends without implementing > `tool github.com/apache/airflow/go-sdk/cmd/airflow-go-pack` to your bundle module's `go.mod` and run > it with `go tool airflow-go-pack`. This pins the packer version per project. -- Register the coordinator and route the queue to it, under `[sdk]` in `airflow.cfg` (or the equivalent - `AIRFLOW__SDK__*` env vars): +- Register a Dag bundle for the packed bundles, register the coordinator, and route the queue to it, in + `airflow.cfg` (or the equivalent `AIRFLOW__*` env vars): ```ini + [dag_processor] + dag_bundle_config_list = [ + {"name": "dags-folder", "classpath": "airflow.dag_processing.bundles.local.LocalDagBundle", "kwargs": {}}, + {"name": "go-task-handlers", "classpath": "airflow.dag_processing.bundles.local.LocalDagBundle", "kwargs": {"path": "/opt/airflow/go-task-handlers"}} + ] + [sdk] - coordinators = {"go": {"classpath": "airflow.sdk.coordinators.executable.ExecutableCoordinator", "kwargs": {"executables_root": ["~/airflow/executable-bundles"]}}} + coordinators = {"go": {"classpath": "airflow.sdk.coordinators.executable.ExecutableCoordinator", "kwargs": {"task_handler_bundle_name": "go-task-handlers"}}} queue_to_coordinator = {"golang": "go"} ``` - `executables_root` is one or more directories the coordinator scans for bundles; `queue_to_coordinator` - routes stub tasks with `queue="golang"` to this Go coordinator. + `task_handler_bundle_name` names the Dag bundle the coordinator scans for packed bundles (the task's own + Dag bundle when unset); `queue_to_coordinator` routes stub tasks with `queue="golang"` to this Go + coordinator. Only files with the executable bit are considered, so use a Dag bundle that keeps it: a + `LocalDagBundle` does, object-store Dag bundles such as `S3DagBundle` do not. > [!IMPORTANT] - > The coordinator is part of the Airflow worker, so the `[sdk]` config (and the bundle files in - > `executables_root`) only need to be present wherever tasks actually execute. With `CeleryExecutor`, - > setting it on the Celery workers is sufficient. With `LocalExecutor`, tasks run inside the scheduler - > process, so it must be set where the scheduler can read it. The API server and Dag processor do not - > need it. + > The coordinator is part of the Airflow worker, so the `[sdk]` config and the packed bundle files only + > need to be present wherever tasks actually execute. With `CeleryExecutor`, setting it on the Celery + > workers is sufficient. With `LocalExecutor`, tasks run inside the scheduler process, so it must be set + > where the scheduler can read it. The API server and Dag processor do not need it. The worker resolves + > `task_handler_bundle_name` through `[dag_processor] dag_bundle_config_list`, so the workers need that + > setting too. - Deploy the matching Python stub Dag (above) into Airflow. There is no separate Go worker to run: the Airflow worker forks the bundle binary once per task instance. diff --git a/go-sdk/example/bundle/Justfile b/go-sdk/example/bundle/Justfile index 5a12c4407aed1..07df82a2280a6 100644 --- a/go-sdk/example/bundle/Justfile +++ b/go-sdk/example/bundle/Justfile @@ -29,8 +29,9 @@ build: pack # DAG/task identity via --airflow-metadata, and appends the source plus # airflow-metadata.yaml plus AFBNDL01 trailer. The output is a single # self-contained executable bundle, named after the bundle's package -# directory and written to the current directory. Drop it into -# ExecutableCoordinator's executables_root to deploy. +# directory and written to the current directory. Drop it into the +# Dag bundle named by ExecutableCoordinator's task_handler_bundle_name +# to deploy. pack: @echo "Packing example DAG bundle..." go tool airflow-go-pack --output ../../bin/example_dags diff --git a/java-sdk/README.md b/java-sdk/README.md index b8600ab5d44fa..5ec0995708546 100644 --- a/java-sdk/README.md +++ b/java-sdk/README.md @@ -96,13 +96,22 @@ Now `cd example` into the example project, and * Ensure the `java` command is available in the same environment the Airflow task worker is in. -* Configure Airflow to route tasks in the *java* queue to be run with Java: +* Register the packaged example as a Dag bundle, and configure Airflow to route tasks in the *java* + queue to be run with Java from it: ```bash + export AIRFLOW__DAG_PROCESSOR__DAG_BUNDLE_CONFIG_LIST='[ + {"name": "dags-folder", "classpath": "airflow.dag_processing.bundles.local.LocalDagBundle", "kwargs": {}}, + { + "name": "java-task-handlers", + "classpath": "airflow.dag_processing.bundles.local.LocalDagBundle", + "kwargs": {"path": "/opt/airflow/java-sdk/example/build/bundle"} + } + ]' export AIRFLOW__SDK__COORDINATORS='{ "java": { "classpath": "airflow.sdk.coordinators.java.JavaCoordinator", - "kwargs": {"jars_root": ["/opt/airflow/java-sdk/example/build/bundle"]} + "kwargs": {"task_handler_bundle_name": "java-task-handlers"} } }' export AIRFLOW__SDK__QUEUE_TO_COORDINATOR='{"java": "java"}' @@ -644,9 +653,9 @@ where Airflow can find it. When the Airflow supervisor identifies that a task should run with Java, it launches the JVM application as a subprocess. The flow is: -1. `JavaCoordinator.execute_task()` (Python) scans `jars_root`, builds the - classpath, and spawns `java -cp --comm=: - --logs=:`. +1. `JavaCoordinator.execute_task()` (Python) scans the Dag bundle named by + `task_handler_bundle_name`, builds the classpath, and spawns + `java -cp --comm=: --logs=:`. 2. `Server.kt` connects to both sockets immediately on startup. 3. The supervisor sends a `StartupDetails` MessagePack message; the JVM reads it, looks up the matching task by `dag_id` + `task_id`, and calls the diff --git a/java-sdk/plugin/src/main/kotlin/org/apache/airflow/sdk/plugin/AirflowSdkPlugin.kt b/java-sdk/plugin/src/main/kotlin/org/apache/airflow/sdk/plugin/AirflowSdkPlugin.kt index 95836c12fc5fe..3756ef688e174 100644 --- a/java-sdk/plugin/src/main/kotlin/org/apache/airflow/sdk/plugin/AirflowSdkPlugin.kt +++ b/java-sdk/plugin/src/main/kotlin/org/apache/airflow/sdk/plugin/AirflowSdkPlugin.kt @@ -83,8 +83,8 @@ abstract class AirflowBundleExtension { * * The plugin automatically sets the `Main-Class` metadata, and provides a new * task `bundle` to create a Dag bundle in one command. This builds deploy-ready - * artifacts to `build/bundle/` that can be copied directly into an Airflow Java - * coordinator's `jars_root`. + * artifacts to `build/bundle/` that can be copied directly into the Dag bundle + * named by an Airflow Java coordinator's `task_handler_bundle_name`. * * By default, plugin `com.github.johnrengelman.shadow` is applied automatically * to enable fat JAR build. In this mode, one single JAR with user code and all diff --git a/java-sdk/scala_spark_example/README.md b/java-sdk/scala_spark_example/README.md index f59c01e0e52c6..f736ecd08ee2b 100644 --- a/java-sdk/scala_spark_example/README.md +++ b/java-sdk/scala_spark_example/README.md @@ -41,7 +41,9 @@ cd scala_spark_example ``` `fatJar` is disabled, so `build/bundle/` holds the bundle JAR plus every runtime -JAR (Spark included) — copy it into a Java coordinator's `jars_root`. +JAR (Spark included) — copy it into the Dag bundle named by a Java coordinator's +`task_handler_bundle_name`. Every JAR in that Dag bundle goes on one classpath, +so give Spark its own Dag bundle, coordinator and queue. ## Running Spark under the Java SDK @@ -57,7 +59,7 @@ and bypasses that launcher, so the coordinator has to pass them itself via "scala-jdk": { "classpath": "airflow.sdk.coordinators.java.JavaCoordinator", "kwargs": { - "jars_root": ["/path/to/scala-jars"], + "task_handler_bundle_name": "scala-task-handlers", "main_class": "org.apache.airflow.example.ScalaSparkBundleBuilder", "jvm_args": [ "-Xmx512m", diff --git a/task-sdk/docs/executable-bundle-spec.rst b/task-sdk/docs/executable-bundle-spec.rst index 856aeca94e922..f83443741b101 100644 --- a/task-sdk/docs/executable-bundle-spec.rst +++ b/task-sdk/docs/executable-bundle-spec.rst @@ -283,8 +283,8 @@ Code Signing The bundle format itself does not require OS-level code signing. ``binary_sha256`` provides integrity against truncation, in-flight -corruption, and naive tampering, and Airflow's threat model treats -``executables_root`` as Deployment-Manager-controlled — *authenticity* +corruption, and naive tampering, and Airflow's threat model treats the Dag +bundle holding the bundles as Deployment-Manager-controlled — *authenticity* (signed by a trusted identity) is a deployment-time concern rather than a bundle-format one. @@ -299,22 +299,23 @@ yields a byte-identical bundle file (and therefore a stable Deployment Layout ----------------- -Bundle files are placed **as-is** in any of the directories configured as the -``executables_root`` kwarg on the +Bundle files are placed **as-is** in the Dag bundle named by the +``task_handler_bundle_name`` kwarg on the :class:`~airflow.sdk.coordinators.executable.ExecutableCoordinator` entry -under ``[sdk] coordinators``. The scanner walks each directory **recursively** -and considers only regular files whose **executable bit is set** for the -invoking user; files without the executable bit are skipped without reading -their trailer. For each candidate it reads the last 64 bytes and treats files -whose magic matches ``"AFBNDL01"`` as bundles. Matched files are then -SHA-256-verified per the reader algorithm; a mismatch demotes the file back -to "ignored, with an error log." Files without the magic are silently -ignored, so non-bundle files (READMEs, dotfiles) MAY share the directory +under ``[sdk] coordinators`` (or, when it is unset, in the task's own Dag +bundle). The scanner walks the Dag bundle **recursively** and considers only +regular files whose **executable bit is set** for the invoking user; files +without the executable bit are skipped without reading their trailer, so the +Dag bundle must preserve it. For each candidate it reads the last 64 bytes and +treats files whose magic matches ``"AFBNDL01"`` as bundles. Matched files are +then SHA-256-verified per the reader algorithm; a mismatch demotes the file +back to "ignored, with an error log." Files without the magic are silently +ignored, so non-bundle files (READMEs, dotfiles) MAY share the Dag bundle without interfering with the scan. :: - /opt/airflow/executable-bundles/ + /opt/airflow/go-task-handlers/ ├── example ├── team-a/ │ └── pipeline diff --git a/task-sdk/docs/ts-bundle-spec.rst b/task-sdk/docs/ts-bundle-spec.rst index 705227efa55b2..03ee63ceea0a6 100644 --- a/task-sdk/docs/ts-bundle-spec.rst +++ b/task-sdk/docs/ts-bundle-spec.rst @@ -24,8 +24,8 @@ This document specifies the bundle format produced by ``airflow-ts-pack`` and co Artifact Name ------------- -A bundle's name must end in ``.min.mjs``. Nothing else about it is significant: the coordinator searches each -configured root recursively and routes on embedded metadata, so one root may hold several differently named bundles. +A bundle's name must end in ``.min.mjs``. Nothing else about it is significant: the coordinator searches its Dag +bundle recursively and routes on embedded metadata, so one Dag bundle may hold several differently named bundles. ``airflow-ts-pack`` writes ``bundle.min.mjs`` by default and accepts ``--outfile`` for any other name ending in that suffix. @@ -153,11 +153,12 @@ documents, and neither is used to execute the bundle. Reader and Selection Algorithm ------------------------------ -For each candidate in ``bundles_root``, the coordinator: +For each candidate in the Dag bundle named by ``task_handler_bundle_name`` (or, when it is unset, the task's own Dag +bundle), the coordinator: -1. Opens it once. Candidates are the files whose name ends in ``.min.mjs``, found by walking each root recursively, - roots in configured order and each directory's entries in sorted order, so selection does not depend on the order - a filesystem returns entries in. Directories are deduplicated by ``(st_dev, st_ino)``, so a symlink loop +1. Opens it once. Candidates are the files whose name ends in ``.min.mjs``, found by walking the Dag bundle + recursively, each directory's entries in sorted order, so selection does not depend on the order a filesystem + returns entries in. Directories are deduplicated by ``(st_dev, st_ino)``, so a symlink loop terminates the walk instead of exhausting the interpreter stack. 2. Reads a bounded first line and decodes the named metadata and code ranges. 3. Reads the bounded metadata line and checks that the declared metadata range matches its physical location. @@ -171,15 +172,14 @@ For each candidate in ``bundles_root``, the coordinator: 9. Resolves the supervisor schema version and selects the first usable match. A missing, unrelated, unreadable, malformed, corrupt, or incompatible earlier candidate does not prevent selection -of a later usable match. When more than one usable bundle declares the same Dag, the first configured match wins. If -none matches, the error identifies the requested Dag, searched roots, and rejected candidates. +of a later usable match. When more than one usable bundle declares the same Dag, the first match in walk order wins. +If none matches, the error identifies the requested Dag, the searched directory, and rejected candidates. -Every ``.min.mjs`` file under a root is therefore opened, and one that is not a usable bundle is named among those -rejected candidates. ``bundles_root`` names directories of deployed Airflow bundles, so unrelated minified modules do -not belong there. +Every ``.min.mjs`` file in the Dag bundle is therefore opened, and one that is not a usable bundle is named among +those rejected candidates. That Dag bundle holds deployed Airflow bundles, so unrelated minified modules do not belong +there. -The coordinator does not cache Dag-to-path routing. It checks root ordering and the current deployed files for each -task selection. It may reuse section digests from a bounded process-local cache when the open file identity, +The coordinator does not cache Dag-to-path routing. It checks the current deployed files for each task selection. It may reuse section digests from a bounded process-local cache when the open file identity, timestamps, size, layout ranges, and declared digests have not changed. Integrity, Authenticity, and Provenance diff --git a/ts-sdk/README.md b/ts-sdk/README.md index 361e987fa3116..75c5859b5ca8f 100644 --- a/ts-sdk/README.md +++ b/ts-sdk/README.md @@ -88,18 +88,29 @@ sales_pipeline() Airflow coordinator config: ```ini +[dag_processor] +dag_bundle_config_list = [ + {"name": "dags-folder", "classpath": "airflow.dag_processing.bundles.local.LocalDagBundle", "kwargs": {}}, + { + "name": "ts-task-handlers", + "classpath": "airflow.dag_processing.bundles.local.LocalDagBundle", + "kwargs": {"path": "/opt/airflow/ts-bundles"} + } + ] + [sdk] coordinators = { "ts": { "classpath": "airflow.sdk.coordinators.node.NodeCoordinator", - "kwargs": {"bundles_root": ["/opt/airflow/ts-bundles"]} + "kwargs": {"task_handler_bundle_name": "ts-task-handlers"} } } queue_to_coordinator = {"typescript": "ts"} ``` -Each configured bundle directory is searched recursively for `*.min.mjs` bundles built with `airflow-ts-pack` -(see [Packing bundles](#packing-bundles)), which embeds the Airflow metadata in the bundle itself. +The Dag bundle named by `task_handler_bundle_name` (the task's own Dag bundle when unset) is searched +recursively for `*.min.mjs` bundles built with `airflow-ts-pack` (see [Packing bundles](#packing-bundles)), +which embeds the Airflow metadata in the bundle itself. TypeScript entrypoint: diff --git a/ts-sdk/example/README.md b/ts-sdk/example/README.md index 94c2ead395110..85b5c74099f80 100644 --- a/ts-sdk/example/README.md +++ b/ts-sdk/example/README.md @@ -58,14 +58,22 @@ ts-sdk/example/dist/ ## Airflow Configuration -Configure Airflow to route the `typescript` queue to the Node coordinator and -point it at the example bundle directory: +Register the example bundle directory as a Dag bundle, and configure Airflow to +route the `typescript` queue to the Node coordinator that reads it: ```bash +export AIRFLOW__DAG_PROCESSOR__DAG_BUNDLE_CONFIG_LIST='[ + {"name": "dags-folder", "classpath": "airflow.dag_processing.bundles.local.LocalDagBundle", "kwargs": {}}, + { + "name": "ts-task-handlers", + "classpath": "airflow.dag_processing.bundles.local.LocalDagBundle", + "kwargs": {"path": "/absolute/path/to/airflow/ts-sdk/example/dist"} + } +]' export AIRFLOW__SDK__COORDINATORS='{ "ts": { "classpath": "airflow.sdk.coordinators.node.NodeCoordinator", - "kwargs": {"bundles_root": ["/absolute/path/to/airflow/ts-sdk/example/dist"]} + "kwargs": {"task_handler_bundle_name": "ts-task-handlers"} } }' export AIRFLOW__SDK__QUEUE_TO_COORDINATOR='{"typescript": "ts"}' From b9cde7fe612e1d6a84a301cc63fedaa256840a3c Mon Sep 17 00:00:00 2001 From: LIU ZHE YOU Date: Wed, 30 Sep 2026 09:25:19 +0000 Subject: [PATCH 04/25] Tidy names and mocks in the artifact bundle tests Review flagged a Java test fixture still named after the removed jars_root kwarg, an e2e helper without a verb, and two execute_task patches that were not autospecced like the ones beside them. --- .../tests/airflow_e2e_tests/conftest.py | 8 ++++---- .../task_sdk/coordinators/java/test_coordinator.py | 12 ++++++------ .../tests/task_sdk/coordinators/test_subprocess.py | 4 ++-- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/airflow-e2e-tests/tests/airflow_e2e_tests/conftest.py b/airflow-e2e-tests/tests/airflow_e2e_tests/conftest.py index db08b27b06397..1282326cc9a1c 100644 --- a/airflow-e2e-tests/tests/airflow_e2e_tests/conftest.py +++ b/airflow-e2e-tests/tests/airflow_e2e_tests/conftest.py @@ -293,7 +293,7 @@ def _setup_xcom_object_storage_integration(dot_env_file, tmp_dir): ] -def _dag_bundle_config_with_artifacts(artifact_bundles: dict[str, str]) -> str: +def _build_dag_bundle_config(artifact_bundles: dict[str, str]) -> str: """Return a ``dag_bundle_config_list`` of the Dags folder plus a ``LocalDagBundle`` per artifact path. Registration is what makes a coordinator's ``task_handler_bundle_name`` resolvable on the @@ -476,7 +476,7 @@ def _setup_java_sdk_integration(dot_env_file, tmp_dir): # main_class (Spark's large classpath makes Main-Class discovery ambiguous) # and carries Spark's Java 17 module openings, a small driver heap, and a # longer startup timeout for its large dependency classpath. - dag_bundle_config = _dag_bundle_config_with_artifacts( + dag_bundle_config = _build_dag_bundle_config( { "java-task-handlers": "/opt/airflow/java-jars", "scala-task-handlers": "/opt/airflow/scala-jars", @@ -650,7 +650,7 @@ def _setup_go_sdk_integration(dot_env_file, tmp_dir): # Coordinator registry: maps the logical name "go-sdk" to ExecutableCoordinator, # which scans the go-task-handlers Dag bundle for the packed bundle by dag_id. # Queue mapping: routes tasks on the "golang" queue to "go-sdk". - dag_bundle_config = _dag_bundle_config_with_artifacts({"go-task-handlers": "/opt/airflow/go-bundles"}) + dag_bundle_config = _build_dag_bundle_config({"go-task-handlers": "/opt/airflow/go-bundles"}) coordinator_config = json.dumps( { "go-sdk": { @@ -756,7 +756,7 @@ def _setup_ts_sdk_integration(dot_env_file, tmp_dir): for dag_file in ("typescript_example.py", "typescript_taskflow_example.py"): copyfile(TS_SDK_EXAMPLE_PATH / "dags" / dag_file, tmp_dir / "dags" / dag_file) - dag_bundle_config = _dag_bundle_config_with_artifacts({"ts-task-handlers": "/opt/airflow/ts-bundles"}) + dag_bundle_config = _build_dag_bundle_config({"ts-task-handlers": "/opt/airflow/ts-bundles"}) coordinator_config = json.dumps( { "ts": { diff --git a/task-sdk/tests/task_sdk/coordinators/java/test_coordinator.py b/task-sdk/tests/task_sdk/coordinators/java/test_coordinator.py index eda377e6251fd..56e21d970e612 100644 --- a/task-sdk/tests/task_sdk/coordinators/java/test_coordinator.py +++ b/task-sdk/tests/task_sdk/coordinators/java/test_coordinator.py @@ -252,18 +252,18 @@ def test_build_command_scans_passed_roots_in_colocated_mode(self, tmp_path): @pytest.fixture -def jars_root(tmp_path): +def jars_dir(tmp_path): _make_jar(tmp_path.joinpath("app.jar"), main_class="com.example.TaskRunner", schema_version="2026-06-16") return tmp_path @pytest.fixture -def java_task_handlers(jars_root): - """Register *jars_root* as the ``java-task-handlers`` Dag bundle and return that name.""" +def java_task_handlers(jars_dir): + """Register *jars_dir* as the ``java-task-handlers`` Dag bundle and return that name.""" bundle = { "name": "java-task-handlers", "classpath": "airflow.dag_processing.bundles.local.LocalDagBundle", - "kwargs": {"path": str(jars_root)}, + "kwargs": {"path": str(jars_dir)}, } with conf_vars({("dag_processor", "dag_bundle_config_list"): json.dumps([bundle])}): yield bundle["name"] @@ -337,12 +337,12 @@ def test_java_executable_is_first_arg(self, java_task_handlers, mock_client): ) assert cmd[0] == "/usr/lib/jvm/java-17/bin/java" - def test_classpath_flag_and_value_present(self, jars_root, java_task_handlers, mock_client): + def test_classpath_flag_and_value_present(self, jars_dir, java_task_handlers, mock_client): cmd = self._captured_popen_cmd(java_task_handlers, mock_client) assert "-classpath" in cmd cp_idx = cmd.index("-classpath") classpath = cmd[cp_idx + 1] - assert jars_root.joinpath("app.jar").as_posix() in classpath + assert jars_dir.joinpath("app.jar").as_posix() in classpath def test_main_class_present(self, java_task_handlers, mock_client): cmd = self._captured_popen_cmd(java_task_handlers, mock_client) diff --git a/task-sdk/tests/task_sdk/coordinators/test_subprocess.py b/task-sdk/tests/task_sdk/coordinators/test_subprocess.py index 909a2c605f0ba..f836c6797350c 100644 --- a/task-sdk/tests/task_sdk/coordinators/test_subprocess.py +++ b/task-sdk/tests/task_sdk/coordinators/test_subprocess.py @@ -937,7 +937,7 @@ class TestExecuteTaskBundleWiring: @patch("airflow.sdk.coordinators._subprocess.BundleVersionLock", autospec=True) @patch("airflow.sdk.coordinators._subprocess.initialize_ti_bundle", autospec=True) - @patch.object(_PopenActivitySubprocess, "start") + @patch.object(_PopenActivitySubprocess, "start", autospec=True) def test_task_bundle_mode_binds_forwards_roots_and_locks( self, mock_start, mock_initialize, mock_lock, mock_client, tmp_path ): @@ -963,7 +963,7 @@ def test_task_bundle_mode_binds_forwards_roots_and_locks( @patch("airflow.sdk.coordinators._subprocess.BundleVersionLock", autospec=True) @patch("airflow.sdk.coordinators._subprocess.initialize_ti_bundle", autospec=True) - @patch.object(_PopenActivitySubprocess, "start") + @patch.object(_PopenActivitySubprocess, "start", autospec=True) def test_named_bundle_mode_locks_the_tree_it_scans( self, mock_start, mock_initialize, mock_lock, mock_client, tmp_path ): From 23898f0586c10e539981a881187ebad9a0880d9f Mon Sep 17 00:00:00 2001 From: LIU ZHE YOU Date: Wed, 30 Sep 2026 09:26:23 +0000 Subject: [PATCH 05/25] Say artifact Dag bundles are registered on every component The guides said only the workers need the artifact Dag bundle registered, which reads as permission to register it there alone. The [sdk] configuration is also loaded by the scheduler under KubernetesExecutor, and it is rejected wherever the name is missing, so the bundle belongs in the shared bundle list. --- .../docs/authoring-and-scheduling/language-sdks/go.rst | 7 ++++--- .../docs/authoring-and-scheduling/language-sdks/java.rst | 9 +++++---- .../language-sdks/typescript.rst | 7 ++++--- go-sdk/README.md | 7 ++++--- 4 files changed, 17 insertions(+), 13 deletions(-) diff --git a/airflow-core/docs/authoring-and-scheduling/language-sdks/go.rst b/airflow-core/docs/authoring-and-scheduling/language-sdks/go.rst index b0599035f9601..d3b1f7743df7e 100644 --- a/airflow-core/docs/authoring-and-scheduling/language-sdks/go.rst +++ b/airflow-core/docs/authoring-and-scheduling/language-sdks/go.rst @@ -213,9 +213,10 @@ There is no separate Go worker to run: the Airflow worker forks the bundle binar The coordinator is part of the Airflow worker, so the ``[sdk]`` config and the packed bundle files only need to be present wherever tasks actually execute. With ``CeleryExecutor``, setting it on the Celery workers is sufficient. With ``LocalExecutor``, tasks run inside the scheduler process, so it must be set - where the scheduler can read it. The API server and Dag processor do not need it. The worker resolves - ``task_handler_bundle_name`` through ``[dag_processor] dag_bundle_config_list``, so the workers need that - setting too. + where the scheduler can read it. The API server and Dag processor do not need it. Register the Dag + bundle in ``[dag_processor] dag_bundle_config_list`` on every component, like your other Dag bundles: + the worker resolves ``task_handler_bundle_name`` through it, and wherever the ``[sdk]`` config is read + it is rejected if the name is missing there. Writing tasks ------------- diff --git a/airflow-core/docs/authoring-and-scheduling/language-sdks/java.rst b/airflow-core/docs/authoring-and-scheduling/language-sdks/java.rst index 5f875617e57e9..36c5f12243674 100644 --- a/airflow-core/docs/authoring-and-scheduling/language-sdks/java.rst +++ b/airflow-core/docs/authoring-and-scheduling/language-sdks/java.rst @@ -278,10 +278,11 @@ located. Restart the affected Airflow components after changing this configuration. The coordinator config and JARs must be available wherever tasks execute. With ``CeleryExecutor``, that means the Celery workers; with -``LocalExecutor``, tasks run in subprocesses on the scheduler's host. The worker resolves -``task_handler_bundle_name`` through ``[dag_processor] dag_bundle_config_list``, so the workers need that -setting too. The API server and Dag processor do not need the JARs, while the Dag processor must receive -``sales_pipeline.py`` through the separate Dag delivery process. +``LocalExecutor``, tasks run in subprocesses on the scheduler's host. Register the Dag bundle in +``[dag_processor] dag_bundle_config_list`` on every component, like your other Dag bundles: the worker +resolves ``task_handler_bundle_name`` through it, and wherever the ``[sdk]`` config is read it is rejected if +the name is missing there. The API server and Dag processor do not need the JARs, while the Dag processor must +receive ``sales_pipeline.py`` through the separate Dag delivery process. After Airflow has parsed the Dag, trigger it from the UI or command line: diff --git a/airflow-core/docs/authoring-and-scheduling/language-sdks/typescript.rst b/airflow-core/docs/authoring-and-scheduling/language-sdks/typescript.rst index 452a9dca2c68c..dfb7ed7d636e7 100644 --- a/airflow-core/docs/authoring-and-scheduling/language-sdks/typescript.rst +++ b/airflow-core/docs/authoring-and-scheduling/language-sdks/typescript.rst @@ -246,9 +246,10 @@ task instance. The coordinator runs inside the Airflow worker, so the ``[sdk]`` config and the packed ``*.min.mjs`` bundles only need to be present wherever tasks actually execute. With ``CeleryExecutor``, setting them on the Celery workers is sufficient. With ``LocalExecutor``, tasks run inside the scheduler process, so they - must be present where the scheduler can read them. The API server and Dag processor do not need them. The - worker resolves ``task_handler_bundle_name`` through ``[dag_processor] dag_bundle_config_list``, so the - workers need that setting too. + must be present where the scheduler can read them. The API server and Dag processor do not need them. + Register the Dag bundle in ``[dag_processor] dag_bundle_config_list`` on every component, like your other + Dag bundles: the worker resolves ``task_handler_bundle_name`` through it, and wherever the ``[sdk]`` config + is read it is rejected if the name is missing there. .. _typescript-sdk/native-dag: diff --git a/go-sdk/README.md b/go-sdk/README.md index 5e8dba63ea414..da477119b0c38 100644 --- a/go-sdk/README.md +++ b/go-sdk/README.md @@ -304,9 +304,10 @@ the full range of task states, and alternate XCom backends without implementing > The coordinator is part of the Airflow worker, so the `[sdk]` config and the packed bundle files only > need to be present wherever tasks actually execute. With `CeleryExecutor`, setting it on the Celery > workers is sufficient. With `LocalExecutor`, tasks run inside the scheduler process, so it must be set - > where the scheduler can read it. The API server and Dag processor do not need it. The worker resolves - > `task_handler_bundle_name` through `[dag_processor] dag_bundle_config_list`, so the workers need that - > setting too. + > where the scheduler can read it. The API server and Dag processor do not need it. Register the Dag + > bundle in `[dag_processor] dag_bundle_config_list` on every component, like your other Dag bundles: + > the worker resolves `task_handler_bundle_name` through it, and wherever the `[sdk]` config is read it + > is rejected if the name is missing there. - Deploy the matching Python stub Dag (above) into Airflow. There is no separate Go worker to run: the Airflow worker forks the bundle binary once per task instance. From 8997a357c46d7e90f43cf19ef700d57ec84e2da0 Mon Sep 17 00:00:00 2001 From: LIU ZHE YOU Date: Wed, 30 Sep 2026 10:16:08 +0000 Subject: [PATCH 06/25] Drop em-dashes from the rewritten doc sentences Two sentences rewritten for artifact Dag bundles kept their em-dashes, which the docs style for this work avoids. --- java-sdk/scala_spark_example/README.md | 2 +- task-sdk/docs/executable-bundle-spec.rst | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/java-sdk/scala_spark_example/README.md b/java-sdk/scala_spark_example/README.md index f736ecd08ee2b..34f248a017cf1 100644 --- a/java-sdk/scala_spark_example/README.md +++ b/java-sdk/scala_spark_example/README.md @@ -41,7 +41,7 @@ cd scala_spark_example ``` `fatJar` is disabled, so `build/bundle/` holds the bundle JAR plus every runtime -JAR (Spark included) — copy it into the Dag bundle named by a Java coordinator's +JAR (Spark included). Copy it into the Dag bundle named by a Java coordinator's `task_handler_bundle_name`. Every JAR in that Dag bundle goes on one classpath, so give Spark its own Dag bundle, coordinator and queue. diff --git a/task-sdk/docs/executable-bundle-spec.rst b/task-sdk/docs/executable-bundle-spec.rst index f83443741b101..27e48a0d04fde 100644 --- a/task-sdk/docs/executable-bundle-spec.rst +++ b/task-sdk/docs/executable-bundle-spec.rst @@ -284,9 +284,9 @@ Code Signing The bundle format itself does not require OS-level code signing. ``binary_sha256`` provides integrity against truncation, in-flight corruption, and naive tampering, and Airflow's threat model treats the Dag -bundle holding the bundles as Deployment-Manager-controlled — *authenticity* -(signed by a trusted identity) is a deployment-time concern rather than a -bundle-format one. +bundle holding the bundles as Deployment-Manager-controlled, so +*authenticity* (signed by a trusted identity) is a deployment-time concern +rather than a bundle-format one. **Compressors** such as UPX are NOT supported. They rewrite the file end-to-end, destroying both the trailer and the hash invariant. From 8983f6dc234ff14015dd8791da498036fdf1d46f Mon Sep 17 00:00:00 2001 From: LIU ZHE YOU Date: Thu, 1 Oct 2026 13:45:21 +0000 Subject: [PATCH 07/25] Say the Dag processor needs the language SDK setup too In Airflow 3.4 the Dag processor checks a Python Dag's stub tasks against the task handlers the language SDK artifacts register, so a deployment that follows the old worker-only advice fails that check. task_handler_bundle_name only serves those stub tasks, which the docs did not say. --- .../language-sdks/go.rst | 32 ++++++++++-------- .../language-sdks/index.rst | 6 ++-- .../language-sdks/java.rst | 33 +++++++++++-------- .../language-sdks/typescript.rst | 29 ++++++++-------- .../src/airflow/config_templates/config.yml | 15 ++++++--- go-sdk/README.md | 23 +++++++------ java-sdk/README.md | 11 +++++-- ts-sdk/README.md | 8 ++++- 8 files changed, 96 insertions(+), 61 deletions(-) diff --git a/airflow-core/docs/authoring-and-scheduling/language-sdks/go.rst b/airflow-core/docs/authoring-and-scheduling/language-sdks/go.rst index d3b1f7743df7e..f35012832d6e2 100644 --- a/airflow-core/docs/authoring-and-scheduling/language-sdks/go.rst +++ b/airflow-core/docs/authoring-and-scheduling/language-sdks/go.rst @@ -48,7 +48,8 @@ Prerequisites * Go 1.24 or later to build and pack bundles. This is a build-time requirement only; the worker that runs a packed bundle needs no Go toolchain, because the bundle is a self-contained native executable. -* The packed bundle must be accessible from the Airflow worker, in the Dag bundle the coordinator scans. +* The packed bundle must be accessible from the Airflow worker and the Dag processor, in the Dag bundle the + coordinator scans, and built for the operating system and CPU architecture of both. * The ``apache-airflow-task-sdk`` package (installed with Airflow) provides the coordinator; no additional Python packages are needed. @@ -210,13 +211,14 @@ There is no separate Go worker to run: the Airflow worker forks the bundle binar .. note:: - The coordinator is part of the Airflow worker, so the ``[sdk]`` config and the packed bundle files only - need to be present wherever tasks actually execute. With ``CeleryExecutor``, setting it on the Celery - workers is sufficient. With ``LocalExecutor``, tasks run inside the scheduler process, so it must be set - where the scheduler can read it. The API server and Dag processor do not need it. Register the Dag - bundle in ``[dag_processor] dag_bundle_config_list`` on every component, like your other Dag bundles: - the worker resolves ``task_handler_bundle_name`` through it, and wherever the ``[sdk]`` config is read - it is rejected if the name is missing there. + The ``[sdk]`` config and the packed bundle files must be present wherever tasks execute and on the Dag + processor. With ``CeleryExecutor``, tasks execute on the Celery workers; with ``LocalExecutor``, they run + inside the scheduler process. The Dag processor checks the stub tasks of each Python Dag against the task + handlers the packed bundles register, so it runs them too and needs bundles built for its operating + system and CPU architecture. The API server does not need any of it. Register the Dag bundle in + ``[dag_processor] dag_bundle_config_list`` on every component, like your other Dag bundles: the worker + and the Dag processor resolve ``task_handler_bundle_name`` through it, and wherever the ``[sdk]`` config + is read it is rejected if the name is missing there. Writing tasks ------------- @@ -466,9 +468,9 @@ coordinator scans (see `Deploying`_): Cross-platform builds ~~~~~~~~~~~~~~~~~~~~~~~ -The worker that runs a bundle often uses a different operating system or CPU architecture than your build -machine (for example, deploying to a Linux host from an Apple-silicon ``darwin/arm64`` laptop). Pass -``--goos`` / ``--goarch`` and the packer cross-builds for you: +The worker and the Dag processor that run a bundle often use a different operating system or CPU +architecture than your build machine (for example, deploying to a Linux host from an Apple-silicon +``darwin/arm64`` laptop). Pass ``--goos`` / ``--goarch`` and the packer cross-builds for you: .. code-block:: bash @@ -524,7 +526,9 @@ All ``kwargs`` in the ``coordinators`` config entry are passed to the - Description * - ``task_handler_bundle_name`` - *(task's own Dag bundle)* - - Name of the Dag bundle scanned recursively for executable bundles. It must be registered in + - Name of the Dag bundle scanned recursively for executable bundles. It is used only by + mixed-language Dags, to locate the task handlers for the ``@task.stub`` tasks of a Python Dag; + Dags defined natively in a language SDK do not use it. It must be registered in ``[dag_processor] dag_bundle_config_list``. It is checked when the ``[sdk]`` configuration is loaded, so a typo fails there rather than on the first task. * - ``task_startup_timeout`` @@ -534,8 +538,8 @@ All ``kwargs`` in the ``coordinators`` config entry are passed to the .. note:: - **Locating bundles.** Packed bundles are read from a Dag bundle, so they are delivered, refreshed and - versioned by the same machinery as your Dags. + **Locating bundles.** The packed bundles for the ``@task.stub`` tasks of a Python Dag are read from a Dag + bundle, so they are delivered, refreshed and versioned by the same machinery as your Dags. * The expected layout is a separate Dag bundle for the packed bundles, named by ``task_handler_bundle_name``, rather than the Dag bundle that holds your ``.py`` files. The task diff --git a/airflow-core/docs/authoring-and-scheduling/language-sdks/index.rst b/airflow-core/docs/authoring-and-scheduling/language-sdks/index.rst index 79a0cb3df1f4d..159911d84c9f3 100644 --- a/airflow-core/docs/authoring-and-scheduling/language-sdks/index.rst +++ b/airflow-core/docs/authoring-and-scheduling/language-sdks/index.rst @@ -155,9 +155,9 @@ Coordinators are registered in ``airflow.cfg`` (or via environment variables) un } } - The ``classpath`` value must be importable by the worker. The ``kwargs`` are passed directly - to the coordinator's constructor. See the language-specific guide for the accepted kwargs - of each coordinator (e.g. :ref:`java-sdk/coordinator-config` for + The ``classpath`` value must be importable by the worker and the Dag processor. The ``kwargs`` + are passed directly to the coordinator's constructor. See the language-specific guide for the + accepted kwargs of each coordinator (e.g. :ref:`java-sdk/coordinator-config` for :class:`~airflow.sdk.coordinators.java.JavaCoordinator`). ``extra`` is an optional object for any additional information you want to associate with a diff --git a/airflow-core/docs/authoring-and-scheduling/language-sdks/java.rst b/airflow-core/docs/authoring-and-scheduling/language-sdks/java.rst index 36c5f12243674..eafd542df6955 100644 --- a/airflow-core/docs/authoring-and-scheduling/language-sdks/java.rst +++ b/airflow-core/docs/authoring-and-scheduling/language-sdks/java.rst @@ -41,8 +41,8 @@ Prerequisites * JDK 11 or later is required on the machine that builds the Java project. A local Gradle installation is only needed to generate the Gradle Wrapper for a new project. -* JRE 11 or later must be available on the Airflow worker nodes. -* The compiled task JAR(s) and JVM dependencies must be accessible from the worker. +* JRE 11 or later must be available on the Airflow worker nodes and the Dag processor. +* The compiled task JAR(s) and JVM dependencies must be accessible from the worker and the Dag processor. * The ``apache-airflow-task-sdk`` package (installed with Airflow) provides the coordinator; no additional Python packages are needed. @@ -276,13 +276,15 @@ default, ``${AIRFLOW_HOME}/airflow.cfg``), or set the equivalent ``AIRFLOW__*`` Dag bundle name in ``dag_bundle_config_list``. See :ref:`java-sdk/coordinator-config` for how JARs are located. -Restart the affected Airflow components after changing this configuration. The coordinator config and JARs -must be available wherever tasks execute. With ``CeleryExecutor``, that means the Celery workers; with -``LocalExecutor``, tasks run in subprocesses on the scheduler's host. Register the Dag bundle in -``[dag_processor] dag_bundle_config_list`` on every component, like your other Dag bundles: the worker -resolves ``task_handler_bundle_name`` through it, and wherever the ``[sdk]`` config is read it is rejected if -the name is missing there. The API server and Dag processor do not need the JARs, while the Dag processor must -receive ``sales_pipeline.py`` through the separate Dag delivery process. +Restart the affected Airflow components after changing this configuration. The coordinator config, the JARs +and a JRE must be available wherever tasks execute and on the Dag processor. With ``CeleryExecutor``, tasks +execute on the Celery workers; with ``LocalExecutor``, they run in subprocesses on the scheduler's host. The +Dag processor checks the stub tasks of ``sales_pipeline.py`` against the task handlers the JARs register, so +it runs them too. The API server does not need any of it. Register the Dag bundle in +``[dag_processor] dag_bundle_config_list`` on every component, like your other Dag bundles: the worker and +the Dag processor resolve ``task_handler_bundle_name`` through it, and wherever the ``[sdk]`` config is read +it is rejected if the name is missing there. The Dag processor still receives ``sales_pipeline.py`` through +the separate Dag delivery process. After Airflow has parsed the Dag, trigger it from the UI or command line: @@ -1061,7 +1063,9 @@ All ``kwargs`` in the ``coordinators`` config entry are passed to the - Description * - ``task_handler_bundle_name`` - *(task's own Dag bundle)* - - Name of the Dag bundle scanned recursively for ``.jar`` files. It must be registered in + - Name of the Dag bundle scanned recursively for ``.jar`` files. It is used only by + mixed-language Dags, to locate the task handlers for the ``@task.stub`` tasks of a Python Dag; + Dags defined natively in a language SDK do not use it. It must be registered in ``[dag_processor] dag_bundle_config_list``. It is checked when the ``[sdk]`` configuration is loaded, so a typo fails there rather than on the first task. * - ``java_executable`` @@ -1082,8 +1086,8 @@ All ``kwargs`` in the ``coordinators`` config entry are passed to the .. note:: - **Locating JARs.** JARs are read from a Dag bundle, so they are delivered, refreshed and versioned - by the same machinery as your Dags. + **Locating JARs.** The JARs for the ``@task.stub`` tasks of a Python Dag are read from a Dag bundle, so + they are delivered, refreshed and versioned by the same machinery as your Dags. * The expected layout is a separate Dag bundle for the JARs, named by ``task_handler_bundle_name``, rather than the Dag bundle that holds your ``.py`` files. The task uses the version that Dag bundle @@ -1098,8 +1102,9 @@ All ``kwargs`` in the ``coordinators`` config entry are passed to the The ``[sdk]`` configuration is read at startup, so changes to ``coordinators`` or ``queue_to_coordinator`` (for example adding ``jvm_args``) only take effect after you restart the - scheduler (or ``airflow standalone``). A rebuilt bundle JAR, by contrast, is picked up on the next - task launch without a restart, because a fresh JVM is spawned per task instance. + components that read it: the workers (the scheduler with ``LocalExecutor``), the Dag processor, or + ``airflow standalone``. A rebuilt bundle JAR, by contrast, is picked up on the next task launch without a + restart, because a fresh JVM is spawned per task instance. .. _java-sdk/java-executable: diff --git a/airflow-core/docs/authoring-and-scheduling/language-sdks/typescript.rst b/airflow-core/docs/authoring-and-scheduling/language-sdks/typescript.rst index dfb7ed7d636e7..9364305ee2ddd 100644 --- a/airflow-core/docs/authoring-and-scheduling/language-sdks/typescript.rst +++ b/airflow-core/docs/authoring-and-scheduling/language-sdks/typescript.rst @@ -48,9 +48,9 @@ The SDK is the ``apache-airflow-ts-sdk`` package (ESM-only). It is currently in Prerequisites ------------- -* Node.js 22 or later must be available on the Airflow worker nodes. +* Node.js 22 or later must be available on the Airflow worker nodes and the Dag processor. * The packed bundle (a single ``bundle.min.mjs`` file, see :ref:`typescript-sdk/build`) must be accessible - from the worker, in the Dag bundle the coordinator scans. + from the worker and the Dag processor, in the Dag bundle the coordinator scans. * The ``apache-airflow-task-sdk`` package (installed with Airflow) provides the coordinator; no additional Python packages are needed. * In the TypeScript project, install the ``apache-airflow-ts-sdk`` npm package to author task handlers: @@ -243,13 +243,14 @@ task instance. .. note:: - The coordinator runs inside the Airflow worker, so the ``[sdk]`` config and the packed ``*.min.mjs`` - bundles only need to be present wherever tasks actually execute. With ``CeleryExecutor``, setting them on - the Celery workers is sufficient. With ``LocalExecutor``, tasks run inside the scheduler process, so they - must be present where the scheduler can read them. The API server and Dag processor do not need them. - Register the Dag bundle in ``[dag_processor] dag_bundle_config_list`` on every component, like your other - Dag bundles: the worker resolves ``task_handler_bundle_name`` through it, and wherever the ``[sdk]`` config - is read it is rejected if the name is missing there. + The ``[sdk]`` config, the packed ``*.min.mjs`` bundles and Node.js must be present wherever tasks execute + and on the Dag processor. With ``CeleryExecutor``, tasks execute on the Celery workers; with + ``LocalExecutor``, they run inside the scheduler process. The Dag processor checks the stub tasks of each + Python Dag against the task handlers the packed bundles register, so it runs them too. The API server + does not need any of it. Register the Dag bundle in ``[dag_processor] dag_bundle_config_list`` on every + component, like your other Dag bundles: the worker and the Dag processor resolve + ``task_handler_bundle_name`` through it, and wherever the ``[sdk]`` config is read it is rejected if the + name is missing there. .. _typescript-sdk/native-dag: @@ -544,8 +545,10 @@ All ``kwargs`` in the ``coordinators`` config entry are passed to the * - ``task_handler_bundle_name`` - *(task's own Dag bundle)* - Name of the Dag bundle searched recursively for an integrity-verified ``*.min.mjs`` bundle that - declares the requested Dag. It must be registered in ``[dag_processor] dag_bundle_config_list``. It is - checked when the ``[sdk]`` configuration is loaded, so a typo fails there rather than on the first task. + declares the requested Dag. It is used only by mixed-language Dags, to locate the task handlers for + the ``@task.stub`` tasks of a Python Dag; Dags defined natively in a language SDK do not use it. It + must be registered in ``[dag_processor] dag_bundle_config_list``. It is checked when the ``[sdk]`` + configuration is loaded, so a typo fails there rather than on the first task. * - ``node_executable`` - ``"node"`` - Path to the ``node`` binary. Defaults to ``node`` on ``$PATH``. @@ -556,8 +559,8 @@ All ``kwargs`` in the ``coordinators`` config entry are passed to the .. note:: - **Locating the bundle.** Packed bundles are read from a Dag bundle, so they are delivered, refreshed and - versioned by the same machinery as your Dags. + **Locating the bundle.** The packed bundles for the ``@task.stub`` tasks of a Python Dag are read from a + Dag bundle, so they are delivered, refreshed and versioned by the same machinery as your Dags. * The expected layout is a separate Dag bundle for the packed bundles, named by ``task_handler_bundle_name``, rather than the Dag bundle that holds your ``.py`` files. The task uses diff --git a/airflow-core/src/airflow/config_templates/config.yml b/airflow-core/src/airflow/config_templates/config.yml index 0d93997abe387..eafaecdda739b 100644 --- a/airflow-core/src/airflow/config_templates/config.yml +++ b/airflow-core/src/airflow/config_templates/config.yml @@ -2191,12 +2191,19 @@ sdk: example, two ``JavaCoordinator`` instances pinned to different JDK versions). - The language SDK coordinators read their compiled task handlers from the - Dag bundle named by the ``task_handler_bundle_name`` kwarg. It must name a - bundle in ``[dag_processor] dag_bundle_config_list`` and is checked when - this option is loaded. When it is unset, a task reads its handlers from its + The language SDK coordinators read the task handlers for the + ``@task.stub`` tasks of a Python Dag from the Dag bundle named by the + ``task_handler_bundle_name`` kwarg. Dags defined natively in a language + SDK do not use it. It must name a bundle in + ``[dag_processor] dag_bundle_config_list`` and is checked when this + option is loaded. When it is unset, a task reads its handlers from its own Dag bundle. + The Dag processor needs this option too, with the language runtime the + coordinator starts and the files of that Dag bundle, because it checks + the stub tasks of each Python Dag against the task handlers those files + register. The API server does not need it. + An entry may also carry an optional ``extra`` object for additional information associated with the coordinator that the coordinator itself does not receive; other components read it as needed. For example, diff --git a/go-sdk/README.md b/go-sdk/README.md index da477119b0c38..400726c1acb47 100644 --- a/go-sdk/README.md +++ b/go-sdk/README.md @@ -296,18 +296,21 @@ the full range of task states, and alternate XCom backends without implementing ``` `task_handler_bundle_name` names the Dag bundle the coordinator scans for packed bundles (the task's own - Dag bundle when unset); `queue_to_coordinator` routes stub tasks with `queue="golang"` to this Go - coordinator. Only files with the executable bit are considered, so use a Dag bundle that keeps it: a - `LocalDagBundle` does, object-store Dag bundles such as `S3DagBundle` do not. + Dag bundle when unset). It is used only by mixed-language Dags, to locate the task handlers for the + `@task.stub` tasks of a Python Dag; Dags defined natively in a language SDK do not use it. + `queue_to_coordinator` routes stub tasks with `queue="golang"` to this Go coordinator. Only files with the + executable bit are considered, so use a Dag bundle that keeps it: a `LocalDagBundle` does, object-store + Dag bundles such as `S3DagBundle` do not. > [!IMPORTANT] - > The coordinator is part of the Airflow worker, so the `[sdk]` config and the packed bundle files only - > need to be present wherever tasks actually execute. With `CeleryExecutor`, setting it on the Celery - > workers is sufficient. With `LocalExecutor`, tasks run inside the scheduler process, so it must be set - > where the scheduler can read it. The API server and Dag processor do not need it. Register the Dag - > bundle in `[dag_processor] dag_bundle_config_list` on every component, like your other Dag bundles: - > the worker resolves `task_handler_bundle_name` through it, and wherever the `[sdk]` config is read it - > is rejected if the name is missing there. + > The `[sdk]` config and the packed bundle files must be present wherever tasks execute and on the Dag + > processor. With `CeleryExecutor`, tasks execute on the Celery workers; with `LocalExecutor`, they run + > inside the scheduler process. The Dag processor checks the stub tasks of each Python Dag against the + > task handlers the packed bundles register, so it runs them too and needs bundles built for its + > operating system and CPU architecture. The API server does not need any of it. Register the Dag bundle + > in `[dag_processor] dag_bundle_config_list` on every component, like your other Dag bundles: the worker + > and the Dag processor resolve `task_handler_bundle_name` through it, and wherever the `[sdk]` config is + > read it is rejected if the name is missing there. - Deploy the matching Python stub Dag (above) into Airflow. There is no separate Go worker to run: the Airflow worker forks the bundle binary once per task instance. diff --git a/java-sdk/README.md b/java-sdk/README.md index 5ec0995708546..752253b6ab826 100644 --- a/java-sdk/README.md +++ b/java-sdk/README.md @@ -93,8 +93,8 @@ Now `cd example` into the example project, and * Put the [DAG with stub tasks](./example/src/resources/dags) to somewhere Airflow can find. -* Ensure the `java` command is available in the same environment the Airflow - task worker is in. +* Ensure the `java` command is available in the same environments the Airflow + task worker and the Dag processor are in. * Register the packaged example as a Dag bundle, and configure Airflow to route tasks in the *java* queue to be run with Java from it: @@ -117,6 +117,13 @@ Now `cd example` into the example project, and export AIRFLOW__SDK__QUEUE_TO_COORDINATOR='{"java": "java"}' ``` + Set these, and make the bundle directory available, for the Dag processor + as well as the worker: the Dag processor checks the stub tasks against the + task handlers the JARs register. The API server does not need them. + `task_handler_bundle_name` is used only by mixed-language Dags, to locate + the task handlers for the `@task.stub` tasks of a Python Dag; Dags defined + natively in a language SDK do not use it. + * Ensure the Connection and Variable needed by the example DAG are available: ```bash diff --git a/ts-sdk/README.md b/ts-sdk/README.md index 75c5859b5ca8f..b105f768c6a41 100644 --- a/ts-sdk/README.md +++ b/ts-sdk/README.md @@ -110,7 +110,13 @@ queue_to_coordinator = {"typescript": "ts"} The Dag bundle named by `task_handler_bundle_name` (the task's own Dag bundle when unset) is searched recursively for `*.min.mjs` bundles built with `airflow-ts-pack` (see [Packing bundles](#packing-bundles)), -which embeds the Airflow metadata in the bundle itself. +which embeds the Airflow metadata in the bundle itself. `task_handler_bundle_name` is used only by +mixed-language Dags, to locate the task handlers for the `@task.stub` tasks of a Python Dag; Dags defined +natively in a language SDK do not use it. + +This config, the packed bundles and Node.js 22 or later must be present wherever tasks execute and on the +Dag processor, which checks the stub tasks of each Python Dag against the task handlers the bundles +register. The API server does not need them. TypeScript entrypoint: From c7055408b06e76e24f79c52effba6c13ed859a09 Mon Sep 17 00:00:00 2001 From: ZHE YOU LIU Date: Fri, 2 Oct 2026 11:27:27 +0000 Subject: [PATCH 08/25] Validate task_handler_bundle_name only for routed coordinators A coordinator no queue routes to is never used, so its kwargs are not checked. Add from_config tests for an accepted name, an unrouted coordinator and a spec without the kwarg. --- .../airflow/sdk/execution_time/coordinator.py | 17 +++--- .../execution_time/test_coordinator.py | 56 ++++++++++++++++++- 2 files changed, 63 insertions(+), 10 deletions(-) diff --git a/task-sdk/src/airflow/sdk/execution_time/coordinator.py b/task-sdk/src/airflow/sdk/execution_time/coordinator.py index 8aad5c6899dbc..778ea29aea899 100644 --- a/task-sdk/src/airflow/sdk/execution_time/coordinator.py +++ b/task-sdk/src/airflow/sdk/execution_time/coordinator.py @@ -257,23 +257,22 @@ def from_config(cls) -> Self: """ Load coordinator specs from configuration without initialization. - Every ``queue_to_coordinator`` key and every ``task_handler_bundle_name`` - kwarg is validated here, so a typo fails at config load rather than on the - first task routed to the coordinator. + Every ``queue_to_coordinator`` key and the ``task_handler_bundle_name`` of + every routed coordinator are validated here, so a typo fails at config load + rather than on the first task routed to the coordinator. """ coordinator_specs = { k: _CoordinatorSpec.model_validate(v) for k, v in conf.getjson("sdk", "coordinators", fallback={}).items() } queue_to_coordinator = conf.getjson("sdk", "queue_to_coordinator", fallback={}) - for key in queue_to_coordinator.values(): - if key not in coordinator_specs: + for key in set(queue_to_coordinator.values()): + if (spec := coordinator_specs.get(key)) is None: raise ValueError(f"[sdk] queue_to_coordinator references invalid coordinator key: {key!r}") - for key, spec in coordinator_specs.items(): bundle_name = spec.kwargs.get("task_handler_bundle_name") - if bundle_name is None: - continue - if not isinstance(bundle_name, str) or not DagBundlesManager.is_bundle_configured(bundle_name): + if bundle_name is not None and ( + not isinstance(bundle_name, str) or not DagBundlesManager.is_bundle_configured(bundle_name) + ): raise InvalidCoordinatorError( f"[sdk] coordinators {key!r} sets task_handler_bundle_name={bundle_name!r}, " f"which is not a bundle in [dag_processor] dag_bundle_config_list" diff --git a/task-sdk/tests/task_sdk/execution_time/test_coordinator.py b/task-sdk/tests/task_sdk/execution_time/test_coordinator.py index 3659bce784dbf..ccd6a22280e8d 100644 --- a/task-sdk/tests/task_sdk/execution_time/test_coordinator.py +++ b/task-sdk/tests/task_sdk/execution_time/test_coordinator.py @@ -138,7 +138,7 @@ def test_from_config_rejects_invalid_queue_mapping(self, sdk_config): @pytest.mark.parametrize("bundle_name", ["ghost", ["ghost"]], ids=["unconfigured", "not-a-string"]) def test_from_config_rejects_invalid_task_handler_bundle_name(self, sdk_config, bundle_name): - """Validated for every spec, routed or not, without importing or constructing the coordinator.""" + """Validated without importing or constructing the coordinator.""" sdk_config( coordinators=json.dumps( { @@ -148,10 +148,64 @@ def test_from_config_rejects_invalid_task_handler_bundle_name(self, sdk_config, } } ), + queue_to_coordinator=json.dumps({"queue-boom": "boom"}), ) with pytest.raises(InvalidCoordinatorError, match=r"'boom' sets task_handler_bundle_name="): CoordinatorManager.from_config() + def test_from_config_accepts_configured_task_handler_bundle_name(self, sdk_config): + sdk_config( + coordinators=json.dumps( + { + "boom": { + "classpath": f"{_ExplodingCoordinator.__module__}._ExplodingCoordinator", + "kwargs": {"task_handler_bundle_name": "handlers"}, + } + } + ), + queue_to_coordinator=json.dumps({"queue-boom": "boom"}), + ) + bundles = [ + { + "name": "handlers", + "classpath": "airflow.dag_processing.bundles.local.LocalDagBundle", + "kwargs": {}, + } + ] + with conf_vars({("dag_processor", "dag_bundle_config_list"): json.dumps(bundles)}): + manager = CoordinatorManager.from_config() + assert manager._queue_to_coordinator == {"queue-boom": "boom"} + assert manager._created_coordinators == {} + + def test_from_config_ignores_task_handler_bundle_name_of_unrouted_coordinator(self, sdk_config): + sdk_config( + coordinators=json.dumps( + { + "boom": { + "classpath": f"{_ExplodingCoordinator.__module__}._ExplodingCoordinator", + "kwargs": {"task_handler_bundle_name": "ghost"}, + } + } + ), + ) + manager = CoordinatorManager.from_config() + assert set(manager._coordinator_specs) == {"boom"} + + @mock.patch( + "airflow.sdk.execution_time.coordinator.DagBundlesManager.is_bundle_configured", autospec=True + ) + def test_from_config_skips_bundle_lookup_without_task_handler_bundle_name( + self, is_bundle_configured, sdk_config + ): + sdk_config( + coordinators=json.dumps( + {"alpha": {"classpath": f"{_CoordinatorA.__module__}._CoordinatorA", "kwargs": {}}} + ), + queue_to_coordinator=json.dumps({"queue-a": "alpha"}), + ) + CoordinatorManager.from_config() + is_bundle_configured.assert_not_called() + @pytest.mark.parametrize( ("coordinator_spec", "expected_match"), [ From 9d00ead095fa6ab797d334eacbb84d02c2bd7b0e Mon Sep 17 00:00:00 2001 From: ZHE YOU LIU Date: Fri, 2 Oct 2026 11:27:39 +0000 Subject: [PATCH 09/25] Add newsfragment for the removed coordinator artifact roots --- airflow-core/newsfragments/73970.significant.rst | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 airflow-core/newsfragments/73970.significant.rst diff --git a/airflow-core/newsfragments/73970.significant.rst b/airflow-core/newsfragments/73970.significant.rst new file mode 100644 index 0000000000000..886d1d6ca0a8f --- /dev/null +++ b/airflow-core/newsfragments/73970.significant.rst @@ -0,0 +1,5 @@ +Language SDK coordinators load task handlers from a Dag bundle + +The experimental ``jars_root`` (Java) and ``executables_root`` (Go) coordinator kwargs are removed. +Put the artifacts in a Dag bundle registered in ``[dag_processor] dag_bundle_config_list`` and set ``task_handler_bundle_name`` in the coordinator's ``kwargs``, or leave it unset to use the task's own Dag bundle. +A config that still sets a removed kwarg fails with ``Cannot instantiate coordinator ''``. From 68f03d615b81365140d804b742a9ffaaa6b6c825 Mon Sep 17 00:00:00 2001 From: ZHE YOU LIU Date: Fri, 2 Oct 2026 11:27:44 +0000 Subject: [PATCH 10/25] Mark ADR-0013 as accepted --- .../adr/lang-sdk/0013-persisted-task-handler-bindings.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airflow-core/adr/lang-sdk/0013-persisted-task-handler-bindings.md b/airflow-core/adr/lang-sdk/0013-persisted-task-handler-bindings.md index cbcace7df5231..016f804129ab9 100644 --- a/airflow-core/adr/lang-sdk/0013-persisted-task-handler-bindings.md +++ b/airflow-core/adr/lang-sdk/0013-persisted-task-handler-bindings.md @@ -21,7 +21,7 @@ ## Status -Proposed +Accepted ## Context From c174aa41637140d3b9d233af38a9967ca6810f75 Mon Sep 17 00:00:00 2001 From: LIU ZHE YOU Date: Wed, 30 Sep 2026 11:42:36 +0000 Subject: [PATCH 11/25] Add lang-SDK task handler binding tables ADR-0013 records at parse time which Language SDK artifact runs each stub task, so task execution can stop scanning artifact bundles. This adds the two tables and their migration. Later layers write and read them. The unique key and the owning-file index use an md5 hash of each path instead of the path. A 2000-character path is over MySQL's 3072-byte key limit and Postgres's 2704-byte btree entry limit. This follows DagPriorityParsingRequest. --- airflow-core/docs/migrations-ref.rst | 5 +- ..._3_4_0_add_lang_sdk_task_handler_tables.py | 91 +++++++++++ airflow-core/src/airflow/models/__init__.py | 1 + .../airflow/models/lang_sdk_task_handler.py | 116 ++++++++++++++ airflow-core/src/airflow/utils/db.py | 2 +- .../unit/models/test_lang_sdk_task_handler.py | 147 ++++++++++++++++++ .../tests/unit/utils/test_db_cleanup.py | 2 + 7 files changed, 362 insertions(+), 2 deletions(-) create mode 100644 airflow-core/src/airflow/migrations/versions/0142_3_4_0_add_lang_sdk_task_handler_tables.py create mode 100644 airflow-core/src/airflow/models/lang_sdk_task_handler.py create mode 100644 airflow-core/tests/unit/models/test_lang_sdk_task_handler.py diff --git a/airflow-core/docs/migrations-ref.rst b/airflow-core/docs/migrations-ref.rst index afd3ed01e8c32..aa57aaf1472d2 100644 --- a/airflow-core/docs/migrations-ref.rst +++ b/airflow-core/docs/migrations-ref.rst @@ -39,7 +39,10 @@ Here's the list of all the Database Migrations that are executed via when you ru +-------------------------+------------------+-------------------+--------------------------------------------------------------+ | Revision ID | Revises ID | Airflow Version | Description | +=========================+==================+===================+==============================================================+ -| ``90e4d18ccadf`` (head) | ``e5a91c7f42b3`` | ``3.4.0`` | Add timetable_asset_gated to DagModel. | +| ``ffff41014d89`` (head) | ``90e4d18ccadf`` | ``3.4.0`` | Add lang_sdk_task_handler_artifact and lang_sdk_task_handler | +| | | | tables. | ++-------------------------+------------------+-------------------+--------------------------------------------------------------+ +| ``90e4d18ccadf`` | ``e5a91c7f42b3`` | ``3.4.0`` | Add timetable_asset_gated to DagModel. | +-------------------------+------------------+-------------------+--------------------------------------------------------------+ | ``e5a91c7f42b3`` | ``ca8499dc1004`` | ``3.4.0`` | Add language column to dag_code. | +-------------------------+------------------+-------------------+--------------------------------------------------------------+ diff --git a/airflow-core/src/airflow/migrations/versions/0142_3_4_0_add_lang_sdk_task_handler_tables.py b/airflow-core/src/airflow/migrations/versions/0142_3_4_0_add_lang_sdk_task_handler_tables.py new file mode 100644 index 0000000000000..380e5f1dc2a71 --- /dev/null +++ b/airflow-core/src/airflow/migrations/versions/0142_3_4_0_add_lang_sdk_task_handler_tables.py @@ -0,0 +1,91 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +""" +Add lang_sdk_task_handler_artifact and lang_sdk_task_handler tables. + +Revision ID: ffff41014d89 +Revises: 90e4d18ccadf +Create Date: 2026-09-30 12:00:00.000000 + +""" + +from __future__ import annotations + +import sqlalchemy as sa +from alembic import op + +from airflow.migrations.db_types import StringID +from airflow.utils.sqlalchemy import UtcDateTime + +# revision identifiers, used by Alembic. +revision = "ffff41014d89" +down_revision = "90e4d18ccadf" +branch_labels = None +depends_on = None +airflow_version = "3.4.0" + + +def upgrade(): + """Add lang_sdk_task_handler_artifact and lang_sdk_task_handler tables.""" + op.create_table( + "lang_sdk_task_handler_artifact", + sa.Column("id", sa.Uuid(), nullable=False), + sa.Column("bundle_name", StringID(), nullable=False), + sa.Column("relative_fileloc", sa.String(length=2000), nullable=False), + sa.Column("relative_fileloc_hash", sa.String(length=32), nullable=False), + sa.Column("size_bytes", sa.BigInteger(), nullable=False), + sa.Column("cache_digest", sa.String(length=128), nullable=False), + sa.Column("last_probed_at", UtcDateTime(), nullable=False), + sa.PrimaryKeyConstraint("id", name=op.f("lang_sdk_task_handler_artifact_pkey")), + sa.UniqueConstraint( + "bundle_name", + "relative_fileloc_hash", + name=op.f("lang_sdk_task_handler_artifact_bundle_fileloc_uq"), + ), + ) + op.create_table( + "lang_sdk_task_handler", + sa.Column("dag_id", StringID(), nullable=False), + sa.Column("task_id", StringID(), nullable=False), + sa.Column("artifact_id", sa.Uuid(), nullable=False), + sa.Column("dag_bundle_name", StringID(), nullable=False), + sa.Column("dag_relative_fileloc", sa.String(length=2000), nullable=False), + sa.Column("dag_relative_fileloc_hash", sa.String(length=32), nullable=False), + sa.Column("handler_params", sa.JSON(), nullable=False), + sa.PrimaryKeyConstraint("dag_id", "task_id", name=op.f("lang_sdk_task_handler_pkey")), + sa.ForeignKeyConstraint( + columns=("dag_id",), + refcolumns=["dag.dag_id"], + name="lang_sdk_task_handler_dag_id_fkey", + ondelete="CASCADE", + ), + sa.ForeignKeyConstraint( + columns=("artifact_id",), + refcolumns=["lang_sdk_task_handler_artifact.id"], + name="lang_sdk_task_handler_artifact_id_fkey", + ), + sa.Index("idx_lang_sdk_task_handler_dag_file", "dag_bundle_name", "dag_relative_fileloc_hash"), + sa.Index("idx_lang_sdk_task_handler_artifact_id", "artifact_id"), + ) + + +def downgrade(): + """Drop lang_sdk_task_handler and lang_sdk_task_handler_artifact tables.""" + op.drop_table("lang_sdk_task_handler") + op.drop_table("lang_sdk_task_handler_artifact") diff --git a/airflow-core/src/airflow/models/__init__.py b/airflow-core/src/airflow/models/__init__.py index f5bffb4e57173..f806537c684df 100644 --- a/airflow-core/src/airflow/models/__init__.py +++ b/airflow-core/src/airflow/models/__init__.py @@ -71,6 +71,7 @@ def import_all_models(): import airflow.models.dagwarning import airflow.models.deadline_alert import airflow.models.errors + import airflow.models.lang_sdk_task_handler import airflow.models.revoked_token import airflow.models.serialized_dag import airflow.models.task_state_store diff --git a/airflow-core/src/airflow/models/lang_sdk_task_handler.py b/airflow-core/src/airflow/models/lang_sdk_task_handler.py new file mode 100644 index 0000000000000..5fda05ade2376 --- /dev/null +++ b/airflow-core/src/airflow/models/lang_sdk_task_handler.py @@ -0,0 +1,116 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +from collections.abc import Callable +from datetime import datetime +from typing import TYPE_CHECKING, Any +from uuid import UUID + +import sqlalchemy as sa +import uuid6 +from sqlalchemy import BigInteger, ForeignKeyConstraint, Index, String, UniqueConstraint, Uuid +from sqlalchemy.orm import Mapped, mapped_column, validates + +from airflow._shared.timezones import timezone +from airflow.models.base import Base, StringID +from airflow.utils.hashlib_wrapper import md5 +from airflow.utils.sqlalchemy import UtcDateTime + +if TYPE_CHECKING: + from sqlalchemy.engine.default import DefaultExecutionContext + + +def compute_fileloc_hash(relative_fileloc: str) -> str: + """ + Return the md5 hex digest that the task handler tables index in place of a relative file location. + + A 2000-character path exceeds MySQL's 3072-byte key limit and Postgres's 2704-byte btree entry limit. + """ + return md5(relative_fileloc.encode()).hexdigest() + + +def _build_fileloc_hash_default(path_key: str) -> Callable[[DefaultExecutionContext], str]: + def compute(context: DefaultExecutionContext) -> str: + return compute_fileloc_hash(context.get_current_parameters()[path_key]) + + return compute + + +class LangSDKTaskHandlerArtifact(Base): + """A Language SDK artifact in a task handler Dag bundle, cached with its fingerprint.""" + + __tablename__ = "lang_sdk_task_handler_artifact" + + id: Mapped[UUID] = mapped_column(Uuid(), primary_key=True, default=uuid6.uuid7) + bundle_name: Mapped[str] = mapped_column(StringID(), nullable=False) + relative_fileloc: Mapped[str] = mapped_column(String(2000), nullable=False) + relative_fileloc_hash: Mapped[str] = mapped_column( + String(32), nullable=False, default=_build_fileloc_hash_default("relative_fileloc") + ) + size_bytes: Mapped[int] = mapped_column(BigInteger, nullable=False) + cache_digest: Mapped[str] = mapped_column(String(128), nullable=False) + last_probed_at: Mapped[datetime] = mapped_column(UtcDateTime, nullable=False, default=timezone.utcnow) + + __table_args__ = ( + UniqueConstraint( + bundle_name, + relative_fileloc_hash, + name="lang_sdk_task_handler_artifact_bundle_fileloc_uq", + ), + ) + + +class LangSDKTaskHandler(Base): + """The artifact that runs one stub task, and the task handler parameters it declares.""" + + __tablename__ = "lang_sdk_task_handler" + + dag_id: Mapped[str] = mapped_column(StringID(), primary_key=True) + task_id: Mapped[str] = mapped_column(StringID(), primary_key=True) + artifact_id: Mapped[UUID] = mapped_column(Uuid(), nullable=False) + dag_bundle_name: Mapped[str] = mapped_column(StringID(), nullable=False) + dag_relative_fileloc: Mapped[str] = mapped_column(String(2000), nullable=False) + dag_relative_fileloc_hash: Mapped[str] = mapped_column( + String(32), nullable=False, default=_build_fileloc_hash_default("dag_relative_fileloc") + ) + handler_params: Mapped[list[dict[str, Any]]] = mapped_column(sa.JSON(), nullable=False) + + __table_args__ = ( + ForeignKeyConstraint( + (dag_id,), + ["dag.dag_id"], + name="lang_sdk_task_handler_dag_id_fkey", + ondelete="CASCADE", + ), + # No ON DELETE: deleting an artifact that a handler still references must fail. + ForeignKeyConstraint( + (artifact_id,), + ["lang_sdk_task_handler_artifact.id"], + name="lang_sdk_task_handler_artifact_id_fkey", + ), + Index("idx_lang_sdk_task_handler_dag_file", dag_bundle_name, dag_relative_fileloc_hash), + Index("idx_lang_sdk_task_handler_artifact_id", artifact_id), + ) + + @validates("dag_relative_fileloc") + def _sync_dag_relative_fileloc_hash(self, key: str, dag_relative_fileloc: str) -> str: + # Not an onupdate default: that fires on every UPDATE, even one that leaves the path alone. + # A Core UPDATE or upsert that changes the path must set the hash itself. + self.dag_relative_fileloc_hash = compute_fileloc_hash(dag_relative_fileloc) + return dag_relative_fileloc diff --git a/airflow-core/src/airflow/utils/db.py b/airflow-core/src/airflow/utils/db.py index f160cb0597d84..56adf7b831731 100644 --- a/airflow-core/src/airflow/utils/db.py +++ b/airflow-core/src/airflow/utils/db.py @@ -117,7 +117,7 @@ class MappedClassProtocol(Protocol): "3.1.8": "509b94a1042d", "3.2.0": "1d6611b6ab7c", "3.3.0": "d2f4e1b3c5a7", - "3.4.0": "90e4d18ccadf", + "3.4.0": "ffff41014d89", } # Prefix used to identify tables holding data moved during migration. diff --git a/airflow-core/tests/unit/models/test_lang_sdk_task_handler.py b/airflow-core/tests/unit/models/test_lang_sdk_task_handler.py new file mode 100644 index 0000000000000..52d5d1f033843 --- /dev/null +++ b/airflow-core/tests/unit/models/test_lang_sdk_task_handler.py @@ -0,0 +1,147 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +import pytest +import uuid6 +from sqlalchemy import delete, insert, select +from sqlalchemy.exc import IntegrityError + +from airflow.models.dag import DagModel +from airflow.models.lang_sdk_task_handler import ( + LangSDKTaskHandler, + LangSDKTaskHandlerArtifact, + compute_fileloc_hash, +) + +pytestmark = pytest.mark.db_test + +ARTIFACT_BUNDLE = "java-task-handlers" +# 6000 bytes in UTF-8, over both the MySQL key limit and the Postgres btree entry limit. +LONG_NON_ASCII_PATH = "任" * 2000 + + +def _make_artifact( + *, bundle_name: str = ARTIFACT_BUNDLE, relative_fileloc: str = "etl.jar" +) -> LangSDKTaskHandlerArtifact: + return LangSDKTaskHandlerArtifact( + bundle_name=bundle_name, relative_fileloc=relative_fileloc, size_bytes=1024, cache_digest="0" * 64 + ) + + +def _make_handler(*, dag_id: str, artifact: LangSDKTaskHandlerArtifact) -> LangSDKTaskHandler: + return LangSDKTaskHandler( + dag_id=dag_id, + task_id="extract", + artifact_id=artifact.id, + dag_bundle_name="testing", + dag_relative_fileloc=f"{dag_id}.py", + handler_params=[{"name": "path", "value_schema": None, "required": True}], + ) + + +def _add_dags_and_artifact(session, *dag_ids: str) -> LangSDKTaskHandlerArtifact: + session.add_all(DagModel(dag_id=dag_id, bundle_name="testing") for dag_id in dag_ids) + artifact = _make_artifact() + session.add(artifact) + session.flush() + return artifact + + +def test_deleting_dag_deletes_its_handlers(testing_dag_bundle, session): + artifact = _add_dags_and_artifact(session, "dag_a", "dag_b") + session.add_all(_make_handler(dag_id=dag_id, artifact=artifact) for dag_id in ("dag_a", "dag_b")) + session.flush() + + session.execute(delete(DagModel).where(DagModel.dag_id == "dag_a")) + + assert session.scalars(select(LangSDKTaskHandler.dag_id)).all() == ["dag_b"] + assert session.scalars(select(LangSDKTaskHandlerArtifact.id)).all() == [artifact.id] + + +@pytest.mark.parametrize( + "relative_fileloc", + [ + pytest.param("etl.jar", id="short"), + pytest.param(LONG_NON_ASCII_PATH, id="long-non-ascii"), + ], +) +def test_artifact_path_is_unique_per_bundle(session, relative_fileloc): + session.add_all( + [ + _make_artifact(relative_fileloc=relative_fileloc), + _make_artifact(bundle_name="go-task-handlers", relative_fileloc=relative_fileloc), + ] + ) + session.flush() + + session.add(_make_artifact(relative_fileloc=relative_fileloc)) + with pytest.raises(IntegrityError): + session.flush() + + +def test_core_insert_fills_fileloc_hashes(testing_dag_bundle, session): + session.add(DagModel(dag_id="dag_a", bundle_name="testing")) + session.flush() + artifact_id = uuid6.uuid7() + + session.execute( + insert(LangSDKTaskHandlerArtifact).values( + id=artifact_id, + bundle_name=ARTIFACT_BUNDLE, + relative_fileloc="etl.jar", + size_bytes=1024, + cache_digest="0" * 64, + ) + ) + session.execute( + insert(LangSDKTaskHandler).values( + dag_id="dag_a", + task_id="extract", + artifact_id=artifact_id, + dag_bundle_name="testing", + dag_relative_fileloc="dags/etl.py", + handler_params=[], + ) + ) + + assert session.scalar(select(LangSDKTaskHandlerArtifact.relative_fileloc_hash)) == compute_fileloc_hash( + "etl.jar" + ) + assert session.scalar(select(LangSDKTaskHandler.dag_relative_fileloc_hash)) == compute_fileloc_hash( + "dags/etl.py" + ) + + +def test_dag_relative_fileloc_hash_follows_orm_updates(testing_dag_bundle, session): + artifact = _add_dags_and_artifact(session, "dag_a") + handler = _make_handler(dag_id="dag_a", artifact=artifact) + session.add(handler) + session.flush() + + handler.handler_params = [] + session.flush() + assert session.scalar(select(LangSDKTaskHandler.dag_relative_fileloc_hash)) == compute_fileloc_hash( + "dag_a.py" + ) + + handler.dag_relative_fileloc = "moved/dag_a.py" + session.flush() + assert session.scalar(select(LangSDKTaskHandler.dag_relative_fileloc_hash)) == compute_fileloc_hash( + "moved/dag_a.py" + ) diff --git a/airflow-core/tests/unit/utils/test_db_cleanup.py b/airflow-core/tests/unit/utils/test_db_cleanup.py index 375461b6848ac..41ec3ff7de9a7 100644 --- a/airflow-core/tests/unit/utils/test_db_cleanup.py +++ b/airflow-core/tests/unit/utils/test_db_cleanup.py @@ -1087,6 +1087,7 @@ def test_no_models_missing(self): # leave alone - per-asset key/value state, upserted in place (PK is asset_id+key), # so it is bounded and current, not accumulating history; removed with its asset "asset_state_store", + "lang_sdk_task_handler_artifact", # parse-time cache of task handler artifacts, not run data # Purged indirectly: each of these hangs off a cleaned table by an # ON DELETE CASCADE foreign key, so the rows go when the parent does. # cascade from dag_run once the partition run has fired; while it is still @@ -1098,6 +1099,7 @@ def test_no_models_missing(self): "hitl_detail", # cascade from task_instance "hitl_detail_history", # cascade from task_instance_history "job_team", # cascade from job + "lang_sdk_task_handler", # cascade from dag "task_inlet_asset_reference", # cascade from dag } From 6762729ecfcad603a699edf55fa3b8a7ce7f2fee Mon Sep 17 00:00:00 2001 From: LIU ZHE YOU Date: Wed, 30 Sep 2026 11:45:47 +0000 Subject: [PATCH 12/25] Match ADR-0013 task handler table SQL to the migration The tables as built need an md5 hash column beside each long path, a wider cache_digest, and the repo's constraint names. Later layers read this SQL as the spec, so it now matches the migration. --- .../0013-persisted-task-handler-bindings.md | 41 ++++++++++--------- 1 file changed, 22 insertions(+), 19 deletions(-) diff --git a/airflow-core/adr/lang-sdk/0013-persisted-task-handler-bindings.md b/airflow-core/adr/lang-sdk/0013-persisted-task-handler-bindings.md index 016f804129ab9..d4d242a2152e9 100644 --- a/airflow-core/adr/lang-sdk/0013-persisted-task-handler-bindings.md +++ b/airflow-core/adr/lang-sdk/0013-persisted-task-handler-bindings.md @@ -157,31 +157,34 @@ backs many handlers and its fingerprint must have exactly one value. ```sql CREATE TABLE lang_sdk_task_handler_artifact ( - id UUID NOT NULL, - bundle_name VARCHAR(250) NOT NULL, -- the task_handler_bundle_name it was found in - relative_fileloc VARCHAR(2000) NOT NULL, -- path within that bundle - size_bytes BIGINT NOT NULL, -- cheap fingerprint tier - cache_digest VARCHAR(64) NOT NULL, -- content fingerprint tier; see "The fast path" - last_probed_at TIMESTAMP NOT NULL, - PRIMARY KEY (id), - CONSTRAINT lstha_bundle_fileloc_uq UNIQUE (bundle_name, relative_fileloc) + id UUID NOT NULL, + bundle_name VARCHAR(250) NOT NULL, -- the task_handler_bundle_name it was found in + relative_fileloc VARCHAR(2000) NOT NULL, -- path within that bundle + relative_fileloc_hash VARCHAR(32) NOT NULL, -- md5 of relative_fileloc; the path is too long to index + size_bytes BIGINT NOT NULL, -- cheap fingerprint tier + cache_digest VARCHAR(128) NOT NULL, -- content fingerprint tier; see "The fast path" + last_probed_at TIMESTAMP NOT NULL, + CONSTRAINT lang_sdk_task_handler_artifact_pkey PRIMARY KEY (id), + CONSTRAINT lang_sdk_task_handler_artifact_bundle_fileloc_uq UNIQUE (bundle_name, relative_fileloc_hash) ); CREATE TABLE lang_sdk_task_handler ( - dag_id VARCHAR(250) NOT NULL, - task_id VARCHAR(250) NOT NULL, - artifact_id UUID NOT NULL, - dag_bundle_name VARCHAR(250) NOT NULL, -- the *Python* file that owns this row - dag_relative_fileloc VARCHAR(2000) NOT NULL, -- ditto - handler_params JSON NOT NULL, -- list[TaskHandlerParam], ordered - PRIMARY KEY (dag_id, task_id), - CONSTRAINT lsth_dag_fkey FOREIGN KEY (dag_id) + dag_id VARCHAR(250) NOT NULL, + task_id VARCHAR(250) NOT NULL, + artifact_id UUID NOT NULL, + dag_bundle_name VARCHAR(250) NOT NULL, -- the *Python* file that owns this row + dag_relative_fileloc VARCHAR(2000) NOT NULL, -- ditto + dag_relative_fileloc_hash VARCHAR(32) NOT NULL, -- md5 of dag_relative_fileloc + handler_params JSON NOT NULL, -- list[TaskHandlerParam], ordered + CONSTRAINT lang_sdk_task_handler_pkey PRIMARY KEY (dag_id, task_id), + CONSTRAINT lang_sdk_task_handler_dag_id_fkey FOREIGN KEY (dag_id) REFERENCES dag (dag_id) ON DELETE CASCADE, - CONSTRAINT lsth_artifact_fkey FOREIGN KEY (artifact_id) + CONSTRAINT lang_sdk_task_handler_artifact_id_fkey FOREIGN KEY (artifact_id) REFERENCES lang_sdk_task_handler_artifact (id) ); -CREATE INDEX idx_lsth_dag_file ON lang_sdk_task_handler (dag_bundle_name, dag_relative_fileloc); -CREATE INDEX idx_lsth_artifact_id ON lang_sdk_task_handler (artifact_id); +CREATE INDEX idx_lang_sdk_task_handler_dag_file + ON lang_sdk_task_handler (dag_bundle_name, dag_relative_fileloc_hash); +CREATE INDEX idx_lang_sdk_task_handler_artifact_id ON lang_sdk_task_handler (artifact_id); ``` `PRIMARY KEY (dag_id, task_id)` is the conflict guard: two artifacts claiming the same task cannot From dc15082f1350d02345d14516edb7083a00a09978 Mon Sep 17 00:00:00 2001 From: LIU ZHE YOU Date: Wed, 30 Sep 2026 14:16:32 +0000 Subject: [PATCH 13/25] Keep the artifact path hash in sync on ORM updates An ORM update of an artifact's relative_fileloc kept the old hash, so the unique key no longer matched the path. The handler table already resyncs its hash this way. --- .../airflow/models/lang_sdk_task_handler.py | 10 +++- .../unit/models/test_lang_sdk_task_handler.py | 53 +++++++++++++++---- 2 files changed, 50 insertions(+), 13 deletions(-) diff --git a/airflow-core/src/airflow/models/lang_sdk_task_handler.py b/airflow-core/src/airflow/models/lang_sdk_task_handler.py index 5fda05ade2376..d53301fe05b4a 100644 --- a/airflow-core/src/airflow/models/lang_sdk_task_handler.py +++ b/airflow-core/src/airflow/models/lang_sdk_task_handler.py @@ -45,6 +45,9 @@ def compute_fileloc_hash(relative_fileloc: str) -> str: return md5(relative_fileloc.encode()).hexdigest() +# The column default fills a hash on INSERT, ORM or Core. ORM updates resync it in each model's +# @validates hook instead of an onupdate default, which would fire on every UPDATE, even one that +# leaves the path alone. A Core UPDATE or upsert that changes a path must set its hash itself. def _build_fileloc_hash_default(path_key: str) -> Callable[[DefaultExecutionContext], str]: def compute(context: DefaultExecutionContext) -> str: return compute_fileloc_hash(context.get_current_parameters()[path_key]) @@ -75,6 +78,11 @@ class LangSDKTaskHandlerArtifact(Base): ), ) + @validates("relative_fileloc") + def _sync_relative_fileloc_hash(self, key: str, relative_fileloc: str) -> str: + self.relative_fileloc_hash = compute_fileloc_hash(relative_fileloc) + return relative_fileloc + class LangSDKTaskHandler(Base): """The artifact that runs one stub task, and the task handler parameters it declares.""" @@ -110,7 +118,5 @@ class LangSDKTaskHandler(Base): @validates("dag_relative_fileloc") def _sync_dag_relative_fileloc_hash(self, key: str, dag_relative_fileloc: str) -> str: - # Not an onupdate default: that fires on every UPDATE, even one that leaves the path alone. - # A Core UPDATE or upsert that changes the path must set the hash itself. self.dag_relative_fileloc_hash = compute_fileloc_hash(dag_relative_fileloc) return dag_relative_fileloc diff --git a/airflow-core/tests/unit/models/test_lang_sdk_task_handler.py b/airflow-core/tests/unit/models/test_lang_sdk_task_handler.py index 52d5d1f033843..b99b63bacf70f 100644 --- a/airflow-core/tests/unit/models/test_lang_sdk_task_handler.py +++ b/airflow-core/tests/unit/models/test_lang_sdk_task_handler.py @@ -128,20 +128,51 @@ def test_core_insert_fills_fileloc_hashes(testing_dag_bundle, session): ) -def test_dag_relative_fileloc_hash_follows_orm_updates(testing_dag_bundle, session): - artifact = _add_dags_and_artifact(session, "dag_a") - handler = _make_handler(dag_id="dag_a", artifact=artifact) +def _add_artifact(session) -> LangSDKTaskHandlerArtifact: + artifact = _make_artifact() + session.add(artifact) + session.flush() + return artifact + + +def _add_handler(session) -> LangSDKTaskHandler: + handler = _make_handler(dag_id="dag_a", artifact=_add_dags_and_artifact(session, "dag_a")) session.add(handler) session.flush() + return handler + + +@pytest.mark.parametrize( + ("add_row", "path_attr", "hash_column", "other_attr", "other_value"), + [ + pytest.param( + _add_artifact, + "relative_fileloc", + LangSDKTaskHandlerArtifact.relative_fileloc_hash, + "size_bytes", + 2048, + id="artifact", + ), + pytest.param( + _add_handler, + "dag_relative_fileloc", + LangSDKTaskHandler.dag_relative_fileloc_hash, + "handler_params", + [], + id="handler", + ), + ], +) +def test_fileloc_hash_follows_orm_updates( + testing_dag_bundle, session, add_row, path_attr, hash_column, other_attr, other_value +): + row = add_row(session) + original_path = getattr(row, path_attr) - handler.handler_params = [] + setattr(row, other_attr, other_value) session.flush() - assert session.scalar(select(LangSDKTaskHandler.dag_relative_fileloc_hash)) == compute_fileloc_hash( - "dag_a.py" - ) + assert session.scalar(select(hash_column)) == compute_fileloc_hash(original_path) - handler.dag_relative_fileloc = "moved/dag_a.py" + setattr(row, path_attr, "moved/file") session.flush() - assert session.scalar(select(LangSDKTaskHandler.dag_relative_fileloc_hash)) == compute_fileloc_hash( - "moved/dag_a.py" - ) + assert session.scalar(select(hash_column)) == compute_fileloc_hash("moved/file") From ab8f0257223203028a12e9bbeba836694c8fc446 Mon Sep 17 00:00:00 2001 From: LIU ZHE YOU Date: Wed, 30 Sep 2026 14:18:11 +0000 Subject: [PATCH 14/25] Test that a referenced task handler artifact cannot be deleted The handler-to-artifact foreign key has no ON DELETE on purpose, so a cleanup that races with a new binding fails instead of dropping it. Only the cascade from dag was tested. --- .../tests/unit/models/test_lang_sdk_task_handler.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/airflow-core/tests/unit/models/test_lang_sdk_task_handler.py b/airflow-core/tests/unit/models/test_lang_sdk_task_handler.py index b99b63bacf70f..261a157443167 100644 --- a/airflow-core/tests/unit/models/test_lang_sdk_task_handler.py +++ b/airflow-core/tests/unit/models/test_lang_sdk_task_handler.py @@ -74,6 +74,17 @@ def test_deleting_dag_deletes_its_handlers(testing_dag_bundle, session): assert session.scalars(select(LangSDKTaskHandlerArtifact.id)).all() == [artifact.id] +def test_deleting_referenced_artifact_fails(testing_dag_bundle, session): + artifact = _add_dags_and_artifact(session, "dag_a") + session.add(_make_handler(dag_id="dag_a", artifact=artifact)) + session.flush() + + with pytest.raises(IntegrityError): + session.execute( + delete(LangSDKTaskHandlerArtifact).where(LangSDKTaskHandlerArtifact.id == artifact.id) + ) + + @pytest.mark.parametrize( "relative_fileloc", [ From 5da25f7206e6969fcf3ca1edd394740a4e825d08 Mon Sep 17 00:00:00 2001 From: LIU ZHE YOU Date: Wed, 30 Sep 2026 14:18:53 +0000 Subject: [PATCH 15/25] Describe LangSDKTaskHandler as a stub task binding The old docstring read as if the row were the artifact. A row is the binding of one stub task to an artifact. --- airflow-core/src/airflow/models/lang_sdk_task_handler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airflow-core/src/airflow/models/lang_sdk_task_handler.py b/airflow-core/src/airflow/models/lang_sdk_task_handler.py index d53301fe05b4a..986eacaad9cea 100644 --- a/airflow-core/src/airflow/models/lang_sdk_task_handler.py +++ b/airflow-core/src/airflow/models/lang_sdk_task_handler.py @@ -85,7 +85,7 @@ def _sync_relative_fileloc_hash(self, key: str, relative_fileloc: str) -> str: class LangSDKTaskHandler(Base): - """The artifact that runs one stub task, and the task handler parameters it declares.""" + """The binding of one stub task, by ``(dag_id, task_id)``, to its artifact and declared handler parameters.""" __tablename__ = "lang_sdk_task_handler" From 396cac2600dc7c953ff807645494434bbe5dcdc2 Mon Sep 17 00:00:00 2001 From: LIU ZHE YOU Date: Wed, 30 Sep 2026 14:27:53 +0000 Subject: [PATCH 16/25] Regenerate the task handler migration revision id The id and create date of migration 0142 were not made by alembic. They are now alembic's own rev_id() and a real timestamp, and the hooks updated the revision map and migration reference to match. --- airflow-core/docs/migrations-ref.rst | 2 +- .../versions/0142_3_4_0_add_lang_sdk_task_handler_tables.py | 6 +++--- airflow-core/src/airflow/utils/db.py | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/airflow-core/docs/migrations-ref.rst b/airflow-core/docs/migrations-ref.rst index aa57aaf1472d2..70847aee7d4d3 100644 --- a/airflow-core/docs/migrations-ref.rst +++ b/airflow-core/docs/migrations-ref.rst @@ -39,7 +39,7 @@ Here's the list of all the Database Migrations that are executed via when you ru +-------------------------+------------------+-------------------+--------------------------------------------------------------+ | Revision ID | Revises ID | Airflow Version | Description | +=========================+==================+===================+==============================================================+ -| ``ffff41014d89`` (head) | ``90e4d18ccadf`` | ``3.4.0`` | Add lang_sdk_task_handler_artifact and lang_sdk_task_handler | +| ``f7ed13533d23`` (head) | ``90e4d18ccadf`` | ``3.4.0`` | Add lang_sdk_task_handler_artifact and lang_sdk_task_handler | | | | | tables. | +-------------------------+------------------+-------------------+--------------------------------------------------------------+ | ``90e4d18ccadf`` | ``e5a91c7f42b3`` | ``3.4.0`` | Add timetable_asset_gated to DagModel. | diff --git a/airflow-core/src/airflow/migrations/versions/0142_3_4_0_add_lang_sdk_task_handler_tables.py b/airflow-core/src/airflow/migrations/versions/0142_3_4_0_add_lang_sdk_task_handler_tables.py index 380e5f1dc2a71..ca984a7be2c9c 100644 --- a/airflow-core/src/airflow/migrations/versions/0142_3_4_0_add_lang_sdk_task_handler_tables.py +++ b/airflow-core/src/airflow/migrations/versions/0142_3_4_0_add_lang_sdk_task_handler_tables.py @@ -19,9 +19,9 @@ """ Add lang_sdk_task_handler_artifact and lang_sdk_task_handler tables. -Revision ID: ffff41014d89 +Revision ID: f7ed13533d23 Revises: 90e4d18ccadf -Create Date: 2026-09-30 12:00:00.000000 +Create Date: 2026-09-30 14:19:35.109799 """ @@ -34,7 +34,7 @@ from airflow.utils.sqlalchemy import UtcDateTime # revision identifiers, used by Alembic. -revision = "ffff41014d89" +revision = "f7ed13533d23" down_revision = "90e4d18ccadf" branch_labels = None depends_on = None diff --git a/airflow-core/src/airflow/utils/db.py b/airflow-core/src/airflow/utils/db.py index 56adf7b831731..689d57ded759f 100644 --- a/airflow-core/src/airflow/utils/db.py +++ b/airflow-core/src/airflow/utils/db.py @@ -117,7 +117,7 @@ class MappedClassProtocol(Protocol): "3.1.8": "509b94a1042d", "3.2.0": "1d6611b6ab7c", "3.3.0": "d2f4e1b3c5a7", - "3.4.0": "ffff41014d89", + "3.4.0": "f7ed13533d23", } # Prefix used to identify tables holding data moved during migration. From 61b2d76b38e31edce15a9109ef91509e225da649 Mon Sep 17 00:00:00 2001 From: LIU ZHE YOU Date: Wed, 30 Sep 2026 15:15:51 +0000 Subject: [PATCH 17/25] Store the task handler binding mode beside its params A task handler declaration now says whether it binds stub arguments positionally, by name, or by name or as a whole. The fast path checks a changed Python file against the stored declaration without running the runtime, so it needs the mode as well as the params. --- ..._3_4_0_add_lang_sdk_task_handler_tables.py | 1 + .../airflow/models/lang_sdk_task_handler.py | 1 + .../unit/models/test_lang_sdk_task_handler.py | 21 ++++++++++++++++++- 3 files changed, 22 insertions(+), 1 deletion(-) diff --git a/airflow-core/src/airflow/migrations/versions/0142_3_4_0_add_lang_sdk_task_handler_tables.py b/airflow-core/src/airflow/migrations/versions/0142_3_4_0_add_lang_sdk_task_handler_tables.py index ca984a7be2c9c..a6112e45a5b24 100644 --- a/airflow-core/src/airflow/migrations/versions/0142_3_4_0_add_lang_sdk_task_handler_tables.py +++ b/airflow-core/src/airflow/migrations/versions/0142_3_4_0_add_lang_sdk_task_handler_tables.py @@ -67,6 +67,7 @@ def upgrade(): sa.Column("dag_bundle_name", StringID(), nullable=False), sa.Column("dag_relative_fileloc", sa.String(length=2000), nullable=False), sa.Column("dag_relative_fileloc_hash", sa.String(length=32), nullable=False), + sa.Column("handler_binding", sa.String(length=20), nullable=False), sa.Column("handler_params", sa.JSON(), nullable=False), sa.PrimaryKeyConstraint("dag_id", "task_id", name=op.f("lang_sdk_task_handler_pkey")), sa.ForeignKeyConstraint( diff --git a/airflow-core/src/airflow/models/lang_sdk_task_handler.py b/airflow-core/src/airflow/models/lang_sdk_task_handler.py index 986eacaad9cea..0489e21c1b2f2 100644 --- a/airflow-core/src/airflow/models/lang_sdk_task_handler.py +++ b/airflow-core/src/airflow/models/lang_sdk_task_handler.py @@ -97,6 +97,7 @@ class LangSDKTaskHandler(Base): dag_relative_fileloc_hash: Mapped[str] = mapped_column( String(32), nullable=False, default=_build_fileloc_hash_default("dag_relative_fileloc") ) + handler_binding: Mapped[str] = mapped_column(String(20), nullable=False) handler_params: Mapped[list[dict[str, Any]]] = mapped_column(sa.JSON(), nullable=False) __table_args__ = ( diff --git a/airflow-core/tests/unit/models/test_lang_sdk_task_handler.py b/airflow-core/tests/unit/models/test_lang_sdk_task_handler.py index 261a157443167..4f03fa8008683 100644 --- a/airflow-core/tests/unit/models/test_lang_sdk_task_handler.py +++ b/airflow-core/tests/unit/models/test_lang_sdk_task_handler.py @@ -34,6 +34,10 @@ ARTIFACT_BUNDLE = "java-task-handlers" # 6000 bytes in UTF-8, over both the MySQL key limit and the Postgres btree entry limit. LONG_NON_ASCII_PATH = "任" * 2000 +HANDLER_PARAMS = [ + {"name": "path", "value_schema": {"type": "string"}, "required": True, "exact_name": True}, + {"name": None, "value_schema": None, "required": False, "exact_name": False}, +] def _make_artifact( @@ -51,7 +55,8 @@ def _make_handler(*, dag_id: str, artifact: LangSDKTaskHandlerArtifact) -> LangS artifact_id=artifact.id, dag_bundle_name="testing", dag_relative_fileloc=f"{dag_id}.py", - handler_params=[{"name": "path", "value_schema": None, "required": True}], + handler_binding="positional", + handler_params=HANDLER_PARAMS, ) @@ -74,6 +79,19 @@ def test_deleting_dag_deletes_its_handlers(testing_dag_bundle, session): assert session.scalars(select(LangSDKTaskHandlerArtifact.id)).all() == [artifact.id] +def test_handler_stores_its_declaration(testing_dag_bundle, session): + artifact = _add_dags_and_artifact(session, "dag_a") + session.add(_make_handler(dag_id="dag_a", artifact=artifact)) + session.flush() + session.expire_all() + + stored = session.execute( + select(LangSDKTaskHandler.handler_binding, LangSDKTaskHandler.handler_params) + ).one() + + assert tuple(stored) == ("positional", HANDLER_PARAMS) + + def test_deleting_referenced_artifact_fails(testing_dag_bundle, session): artifact = _add_dags_and_artifact(session, "dag_a") session.add(_make_handler(dag_id="dag_a", artifact=artifact)) @@ -127,6 +145,7 @@ def test_core_insert_fills_fileloc_hashes(testing_dag_bundle, session): artifact_id=artifact_id, dag_bundle_name="testing", dag_relative_fileloc="dags/etl.py", + handler_binding="named_or_whole", handler_params=[], ) ) From fce1160d9b86b435717916d350c01ae8fc910231 Mon Sep 17 00:00:00 2001 From: LIU ZHE YOU Date: Wed, 30 Sep 2026 15:16:54 +0000 Subject: [PATCH 18/25] Add handler_binding to ADR-0013's task handler table SQL The migration now stores the declaration's binding mode, and later layers read this SQL as the spec. --- .../adr/lang-sdk/0013-persisted-task-handler-bindings.md | 1 + 1 file changed, 1 insertion(+) diff --git a/airflow-core/adr/lang-sdk/0013-persisted-task-handler-bindings.md b/airflow-core/adr/lang-sdk/0013-persisted-task-handler-bindings.md index d4d242a2152e9..780a5bc4d7377 100644 --- a/airflow-core/adr/lang-sdk/0013-persisted-task-handler-bindings.md +++ b/airflow-core/adr/lang-sdk/0013-persisted-task-handler-bindings.md @@ -175,6 +175,7 @@ CREATE TABLE lang_sdk_task_handler ( dag_bundle_name VARCHAR(250) NOT NULL, -- the *Python* file that owns this row dag_relative_fileloc VARCHAR(2000) NOT NULL, -- ditto dag_relative_fileloc_hash VARCHAR(32) NOT NULL, -- md5 of dag_relative_fileloc + handler_binding VARCHAR(20) NOT NULL, -- positional | named | named_or_whole handler_params JSON NOT NULL, -- list[TaskHandlerParam], ordered CONSTRAINT lang_sdk_task_handler_pkey PRIMARY KEY (dag_id, task_id), CONSTRAINT lang_sdk_task_handler_dag_id_fkey FOREIGN KEY (dag_id) From d18d0afe556b80105510f04d8c3102a8a664b045 Mon Sep 17 00:00:00 2001 From: LIU ZHE YOU Date: Thu, 1 Oct 2026 06:01:15 +0000 Subject: [PATCH 19/25] Make session keyword-only in the task handler model test helpers airflow-core takes session keyword-only so that it can never be passed in the wrong position, and review flagged the new test helpers for taking it positionally. --- .../unit/models/test_lang_sdk_task_handler.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/airflow-core/tests/unit/models/test_lang_sdk_task_handler.py b/airflow-core/tests/unit/models/test_lang_sdk_task_handler.py index 4f03fa8008683..b696487f0c108 100644 --- a/airflow-core/tests/unit/models/test_lang_sdk_task_handler.py +++ b/airflow-core/tests/unit/models/test_lang_sdk_task_handler.py @@ -60,7 +60,7 @@ def _make_handler(*, dag_id: str, artifact: LangSDKTaskHandlerArtifact) -> LangS ) -def _add_dags_and_artifact(session, *dag_ids: str) -> LangSDKTaskHandlerArtifact: +def _add_dags_and_artifact(*dag_ids: str, session) -> LangSDKTaskHandlerArtifact: session.add_all(DagModel(dag_id=dag_id, bundle_name="testing") for dag_id in dag_ids) artifact = _make_artifact() session.add(artifact) @@ -69,7 +69,7 @@ def _add_dags_and_artifact(session, *dag_ids: str) -> LangSDKTaskHandlerArtifact def test_deleting_dag_deletes_its_handlers(testing_dag_bundle, session): - artifact = _add_dags_and_artifact(session, "dag_a", "dag_b") + artifact = _add_dags_and_artifact("dag_a", "dag_b", session=session) session.add_all(_make_handler(dag_id=dag_id, artifact=artifact) for dag_id in ("dag_a", "dag_b")) session.flush() @@ -80,7 +80,7 @@ def test_deleting_dag_deletes_its_handlers(testing_dag_bundle, session): def test_handler_stores_its_declaration(testing_dag_bundle, session): - artifact = _add_dags_and_artifact(session, "dag_a") + artifact = _add_dags_and_artifact("dag_a", session=session) session.add(_make_handler(dag_id="dag_a", artifact=artifact)) session.flush() session.expire_all() @@ -93,7 +93,7 @@ def test_handler_stores_its_declaration(testing_dag_bundle, session): def test_deleting_referenced_artifact_fails(testing_dag_bundle, session): - artifact = _add_dags_and_artifact(session, "dag_a") + artifact = _add_dags_and_artifact("dag_a", session=session) session.add(_make_handler(dag_id="dag_a", artifact=artifact)) session.flush() @@ -158,15 +158,15 @@ def test_core_insert_fills_fileloc_hashes(testing_dag_bundle, session): ) -def _add_artifact(session) -> LangSDKTaskHandlerArtifact: +def _add_artifact(*, session) -> LangSDKTaskHandlerArtifact: artifact = _make_artifact() session.add(artifact) session.flush() return artifact -def _add_handler(session) -> LangSDKTaskHandler: - handler = _make_handler(dag_id="dag_a", artifact=_add_dags_and_artifact(session, "dag_a")) +def _add_handler(*, session) -> LangSDKTaskHandler: + handler = _make_handler(dag_id="dag_a", artifact=_add_dags_and_artifact("dag_a", session=session)) session.add(handler) session.flush() return handler @@ -196,7 +196,7 @@ def _add_handler(session) -> LangSDKTaskHandler: def test_fileloc_hash_follows_orm_updates( testing_dag_bundle, session, add_row, path_attr, hash_column, other_attr, other_value ): - row = add_row(session) + row = add_row(session=session) original_path = getattr(row, path_attr) setattr(row, other_attr, other_value) From 7b572ab02d506f7a5c4eb2b8cb67a0f212fe094e Mon Sep 17 00:00:00 2001 From: LIU ZHE YOU Date: Thu, 1 Oct 2026 07:04:39 +0000 Subject: [PATCH 20/25] Cache an artifact's task handlers on its row, not on each binding The fast path trusts an artifact's fingerprint as the cache key, but a binding stored only what its own file's probe asked about, so one file probing a rebuilt or colliding artifact left another file's bindings stale. Once the runtime is asked for every handler, its answer depends on the artifact alone, so it belongs beside the fingerprint and is written by the same probe. The binding's own copy then has no reader. --- ..._3_4_0_add_lang_sdk_task_handler_tables.py | 3 +- .../airflow/models/lang_sdk_task_handler.py | 12 ++-- .../unit/models/test_lang_sdk_task_handler.py | 56 ++++++++++++------- 3 files changed, 45 insertions(+), 26 deletions(-) diff --git a/airflow-core/src/airflow/migrations/versions/0142_3_4_0_add_lang_sdk_task_handler_tables.py b/airflow-core/src/airflow/migrations/versions/0142_3_4_0_add_lang_sdk_task_handler_tables.py index a6112e45a5b24..c09bc759f49d9 100644 --- a/airflow-core/src/airflow/migrations/versions/0142_3_4_0_add_lang_sdk_task_handler_tables.py +++ b/airflow-core/src/airflow/migrations/versions/0142_3_4_0_add_lang_sdk_task_handler_tables.py @@ -51,6 +51,7 @@ def upgrade(): sa.Column("relative_fileloc_hash", sa.String(length=32), nullable=False), sa.Column("size_bytes", sa.BigInteger(), nullable=False), sa.Column("cache_digest", sa.String(length=128), nullable=False), + sa.Column("task_handlers", sa.JSON(), nullable=False), sa.Column("last_probed_at", UtcDateTime(), nullable=False), sa.PrimaryKeyConstraint("id", name=op.f("lang_sdk_task_handler_artifact_pkey")), sa.UniqueConstraint( @@ -67,8 +68,6 @@ def upgrade(): sa.Column("dag_bundle_name", StringID(), nullable=False), sa.Column("dag_relative_fileloc", sa.String(length=2000), nullable=False), sa.Column("dag_relative_fileloc_hash", sa.String(length=32), nullable=False), - sa.Column("handler_binding", sa.String(length=20), nullable=False), - sa.Column("handler_params", sa.JSON(), nullable=False), sa.PrimaryKeyConstraint("dag_id", "task_id", name=op.f("lang_sdk_task_handler_pkey")), sa.ForeignKeyConstraint( columns=("dag_id",), diff --git a/airflow-core/src/airflow/models/lang_sdk_task_handler.py b/airflow-core/src/airflow/models/lang_sdk_task_handler.py index 0489e21c1b2f2..f60aaa602ed05 100644 --- a/airflow-core/src/airflow/models/lang_sdk_task_handler.py +++ b/airflow-core/src/airflow/models/lang_sdk_task_handler.py @@ -56,7 +56,12 @@ def compute(context: DefaultExecutionContext) -> str: class LangSDKTaskHandlerArtifact(Base): - """A Language SDK artifact in a task handler Dag bundle, cached with its fingerprint.""" + """ + A Language SDK artifact in a task handler Dag bundle, cached with its fingerprint and probe answer. + + ``task_handlers`` is the runtime's whole answer, ``{dag_id: [TaskHandlerDeclaration as JSON, ...]}``. + It is written only from a probe and together with the fingerprint, so the two always match. + """ __tablename__ = "lang_sdk_task_handler_artifact" @@ -68,6 +73,7 @@ class LangSDKTaskHandlerArtifact(Base): ) size_bytes: Mapped[int] = mapped_column(BigInteger, nullable=False) cache_digest: Mapped[str] = mapped_column(String(128), nullable=False) + task_handlers: Mapped[dict[str, list[dict[str, Any]]]] = mapped_column(sa.JSON(), nullable=False) last_probed_at: Mapped[datetime] = mapped_column(UtcDateTime, nullable=False, default=timezone.utcnow) __table_args__ = ( @@ -85,7 +91,7 @@ def _sync_relative_fileloc_hash(self, key: str, relative_fileloc: str) -> str: class LangSDKTaskHandler(Base): - """The binding of one stub task, by ``(dag_id, task_id)``, to its artifact and declared handler parameters.""" + """The binding of one stub task, by ``(dag_id, task_id)``, to the artifact that registers its handler.""" __tablename__ = "lang_sdk_task_handler" @@ -97,8 +103,6 @@ class LangSDKTaskHandler(Base): dag_relative_fileloc_hash: Mapped[str] = mapped_column( String(32), nullable=False, default=_build_fileloc_hash_default("dag_relative_fileloc") ) - handler_binding: Mapped[str] = mapped_column(String(20), nullable=False) - handler_params: Mapped[list[dict[str, Any]]] = mapped_column(sa.JSON(), nullable=False) __table_args__ = ( ForeignKeyConstraint( diff --git a/airflow-core/tests/unit/models/test_lang_sdk_task_handler.py b/airflow-core/tests/unit/models/test_lang_sdk_task_handler.py index b696487f0c108..f1b549ebc2818 100644 --- a/airflow-core/tests/unit/models/test_lang_sdk_task_handler.py +++ b/airflow-core/tests/unit/models/test_lang_sdk_task_handler.py @@ -17,6 +17,8 @@ # under the License. from __future__ import annotations +from typing import Any + import pytest import uuid6 from sqlalchemy import delete, insert, select @@ -34,17 +36,32 @@ ARTIFACT_BUNDLE = "java-task-handlers" # 6000 bytes in UTF-8, over both the MySQL key limit and the Postgres btree entry limit. LONG_NON_ASCII_PATH = "任" * 2000 -HANDLER_PARAMS = [ - {"name": "path", "value_schema": {"type": "string"}, "required": True, "exact_name": True}, - {"name": None, "value_schema": None, "required": False, "exact_name": False}, -] +TASK_HANDLERS = { + "etl": [ + { + "task_id": "extract", + "binding": "positional", + "params": [ + {"name": "path", "value_schema": {"type": "string"}, "required": True, "exact_name": True}, + {"name": None, "value_schema": None, "required": False, "exact_name": False}, + ], + } + ], +} def _make_artifact( - *, bundle_name: str = ARTIFACT_BUNDLE, relative_fileloc: str = "etl.jar" + *, + bundle_name: str = ARTIFACT_BUNDLE, + relative_fileloc: str = "etl.jar", + task_handlers: dict[str, list[dict[str, Any]]] | None = None, ) -> LangSDKTaskHandlerArtifact: return LangSDKTaskHandlerArtifact( - bundle_name=bundle_name, relative_fileloc=relative_fileloc, size_bytes=1024, cache_digest="0" * 64 + bundle_name=bundle_name, + relative_fileloc=relative_fileloc, + size_bytes=1024, + cache_digest="0" * 64, + task_handlers=TASK_HANDLERS if task_handlers is None else task_handlers, ) @@ -55,8 +72,6 @@ def _make_handler(*, dag_id: str, artifact: LangSDKTaskHandlerArtifact) -> LangS artifact_id=artifact.id, dag_bundle_name="testing", dag_relative_fileloc=f"{dag_id}.py", - handler_binding="positional", - handler_params=HANDLER_PARAMS, ) @@ -79,17 +94,19 @@ def test_deleting_dag_deletes_its_handlers(testing_dag_bundle, session): assert session.scalars(select(LangSDKTaskHandlerArtifact.id)).all() == [artifact.id] -def test_handler_stores_its_declaration(testing_dag_bundle, session): - artifact = _add_dags_and_artifact("dag_a", session=session) - session.add(_make_handler(dag_id="dag_a", artifact=artifact)) +@pytest.mark.parametrize( + "task_handlers", + [ + pytest.param(TASK_HANDLERS, id="handlers"), + pytest.param({}, id="none"), + ], +) +def test_artifact_stores_its_task_handlers(session, task_handlers): + session.add(_make_artifact(task_handlers=task_handlers)) session.flush() session.expire_all() - stored = session.execute( - select(LangSDKTaskHandler.handler_binding, LangSDKTaskHandler.handler_params) - ).one() - - assert tuple(stored) == ("positional", HANDLER_PARAMS) + assert session.scalar(select(LangSDKTaskHandlerArtifact.task_handlers)) == task_handlers def test_deleting_referenced_artifact_fails(testing_dag_bundle, session): @@ -136,6 +153,7 @@ def test_core_insert_fills_fileloc_hashes(testing_dag_bundle, session): relative_fileloc="etl.jar", size_bytes=1024, cache_digest="0" * 64, + task_handlers={}, ) ) session.execute( @@ -145,8 +163,6 @@ def test_core_insert_fills_fileloc_hashes(testing_dag_bundle, session): artifact_id=artifact_id, dag_bundle_name="testing", dag_relative_fileloc="dags/etl.py", - handler_binding="named_or_whole", - handler_params=[], ) ) @@ -187,8 +203,8 @@ def _add_handler(*, session) -> LangSDKTaskHandler: _add_handler, "dag_relative_fileloc", LangSDKTaskHandler.dag_relative_fileloc_hash, - "handler_params", - [], + "dag_bundle_name", + "other-bundle", id="handler", ), ], From 03613039ea9cac9dfe8ef08b0647bd67e7dabc44 Mon Sep 17 00:00:00 2001 From: LIU ZHE YOU Date: Thu, 1 Oct 2026 07:05:11 +0000 Subject: [PATCH 21/25] Move the handler declarations to the artifact in ADR-0013's SQL The migration now caches the probe answer on the artifact row, and later layers read this SQL as the spec. --- .../lang-sdk/0013-persisted-task-handler-bindings.md | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/airflow-core/adr/lang-sdk/0013-persisted-task-handler-bindings.md b/airflow-core/adr/lang-sdk/0013-persisted-task-handler-bindings.md index 780a5bc4d7377..74a93c0e4863c 100644 --- a/airflow-core/adr/lang-sdk/0013-persisted-task-handler-bindings.md +++ b/airflow-core/adr/lang-sdk/0013-persisted-task-handler-bindings.md @@ -163,6 +163,7 @@ CREATE TABLE lang_sdk_task_handler_artifact ( relative_fileloc_hash VARCHAR(32) NOT NULL, -- md5 of relative_fileloc; the path is too long to index size_bytes BIGINT NOT NULL, -- cheap fingerprint tier cache_digest VARCHAR(128) NOT NULL, -- content fingerprint tier; see "The fast path" + task_handlers JSON NOT NULL, -- {dag_id: [TaskHandlerDeclaration, ...]}, the probe answer last_probed_at TIMESTAMP NOT NULL, CONSTRAINT lang_sdk_task_handler_artifact_pkey PRIMARY KEY (id), CONSTRAINT lang_sdk_task_handler_artifact_bundle_fileloc_uq UNIQUE (bundle_name, relative_fileloc_hash) @@ -175,8 +176,6 @@ CREATE TABLE lang_sdk_task_handler ( dag_bundle_name VARCHAR(250) NOT NULL, -- the *Python* file that owns this row dag_relative_fileloc VARCHAR(2000) NOT NULL, -- ditto dag_relative_fileloc_hash VARCHAR(32) NOT NULL, -- md5 of dag_relative_fileloc - handler_binding VARCHAR(20) NOT NULL, -- positional | named | named_or_whole - handler_params JSON NOT NULL, -- list[TaskHandlerParam], ordered CONSTRAINT lang_sdk_task_handler_pkey PRIMARY KEY (dag_id, task_id), CONSTRAINT lang_sdk_task_handler_dag_id_fkey FOREIGN KEY (dag_id) REFERENCES dag (dag_id) ON DELETE CASCADE, @@ -198,11 +197,10 @@ joining through `DagModel`: `dag.relative_fileloc` is not indexed, and the codeb that querying it means "a sequential scan of dag" ([`reassign_dags_with_unconfigured_bundles`](https://github.com/apache/airflow/blob/79991cd4db0c9346a28b23c453377f6df0c6b4ed/airflow-core/src/airflow/dag_processing/bundles/manager.py#L478)). -`handler_params` stores what the runtime declared, so a changed Python file can be re-validated -against a cached declaration with no subprocess. It is deliberately **not** called `arg_bindings`: -that name already denotes the Python side of the comparison (`XComArgBinding` / `LiteralArgBinding`, -carrying wiring and values, [`build_arg_bindings`](https://github.com/apache/airflow/blob/79991cd4db0c9346a28b23c453377f6df0c6b4ed/airflow-core/src/airflow/serialization/stub_arg_bindings.py#L221-L286)), -and reusing it would make the validation read as comparing a thing to itself. +`task_handlers` caches the artifact's whole probe answer, every handler it registers keyed by +`dag_id`, so a changed Python file can be re-validated against it with no subprocess. It is written +only from a probe, together with `size_bytes` and `cache_digest`, so the answer always belongs to the +fingerprint beside it. A binding row is then only a mapping from a stub task to its artifact. `cache_digest` is **opaque and coordinator-defined**, not "SHA-256 of the file". From 2c96b482aa64df25b03864d2b4657abe1411f9df Mon Sep 17 00:00:00 2001 From: LIU ZHE YOU Date: Thu, 1 Oct 2026 07:11:58 +0000 Subject: [PATCH 22/25] Let a task handler artifact store no cache digest Some artifacts carry no digest: a Go bundle packed before digests.cache existed, or a third-party coordinator that computes none. Such a candidate is still probed and bound, so it needs an artifact row. A NULL digest never matches, so the candidate is probed on every parse. --- .../0142_3_4_0_add_lang_sdk_task_handler_tables.py | 2 +- .../src/airflow/models/lang_sdk_task_handler.py | 3 ++- .../tests/unit/models/test_lang_sdk_task_handler.py | 11 ++++++++++- 3 files changed, 13 insertions(+), 3 deletions(-) diff --git a/airflow-core/src/airflow/migrations/versions/0142_3_4_0_add_lang_sdk_task_handler_tables.py b/airflow-core/src/airflow/migrations/versions/0142_3_4_0_add_lang_sdk_task_handler_tables.py index c09bc759f49d9..c9679ede25bec 100644 --- a/airflow-core/src/airflow/migrations/versions/0142_3_4_0_add_lang_sdk_task_handler_tables.py +++ b/airflow-core/src/airflow/migrations/versions/0142_3_4_0_add_lang_sdk_task_handler_tables.py @@ -50,7 +50,7 @@ def upgrade(): sa.Column("relative_fileloc", sa.String(length=2000), nullable=False), sa.Column("relative_fileloc_hash", sa.String(length=32), nullable=False), sa.Column("size_bytes", sa.BigInteger(), nullable=False), - sa.Column("cache_digest", sa.String(length=128), nullable=False), + sa.Column("cache_digest", sa.String(length=128), nullable=True), sa.Column("task_handlers", sa.JSON(), nullable=False), sa.Column("last_probed_at", UtcDateTime(), nullable=False), sa.PrimaryKeyConstraint("id", name=op.f("lang_sdk_task_handler_artifact_pkey")), diff --git a/airflow-core/src/airflow/models/lang_sdk_task_handler.py b/airflow-core/src/airflow/models/lang_sdk_task_handler.py index f60aaa602ed05..86cb020f3f1d7 100644 --- a/airflow-core/src/airflow/models/lang_sdk_task_handler.py +++ b/airflow-core/src/airflow/models/lang_sdk_task_handler.py @@ -72,7 +72,8 @@ class LangSDKTaskHandlerArtifact(Base): String(32), nullable=False, default=_build_fileloc_hash_default("relative_fileloc") ) size_bytes: Mapped[int] = mapped_column(BigInteger, nullable=False) - cache_digest: Mapped[str] = mapped_column(String(128), nullable=False) + # NULL means the artifact stores no fingerprint and is probed again on every parse. + cache_digest: Mapped[str | None] = mapped_column(String(128), nullable=True) task_handlers: Mapped[dict[str, list[dict[str, Any]]]] = mapped_column(sa.JSON(), nullable=False) last_probed_at: Mapped[datetime] = mapped_column(UtcDateTime, nullable=False, default=timezone.utcnow) diff --git a/airflow-core/tests/unit/models/test_lang_sdk_task_handler.py b/airflow-core/tests/unit/models/test_lang_sdk_task_handler.py index f1b549ebc2818..510e2117dd299 100644 --- a/airflow-core/tests/unit/models/test_lang_sdk_task_handler.py +++ b/airflow-core/tests/unit/models/test_lang_sdk_task_handler.py @@ -54,13 +54,14 @@ def _make_artifact( *, bundle_name: str = ARTIFACT_BUNDLE, relative_fileloc: str = "etl.jar", + cache_digest: str | None = "0" * 64, task_handlers: dict[str, list[dict[str, Any]]] | None = None, ) -> LangSDKTaskHandlerArtifact: return LangSDKTaskHandlerArtifact( bundle_name=bundle_name, relative_fileloc=relative_fileloc, size_bytes=1024, - cache_digest="0" * 64, + cache_digest=cache_digest, task_handlers=TASK_HANDLERS if task_handlers is None else task_handlers, ) @@ -109,6 +110,14 @@ def test_artifact_stores_its_task_handlers(session, task_handlers): assert session.scalar(select(LangSDKTaskHandlerArtifact.task_handlers)) == task_handlers +def test_artifact_can_store_no_cache_digest(session): + session.add(_make_artifact(cache_digest=None)) + session.flush() + session.expire_all() + + assert session.scalar(select(LangSDKTaskHandlerArtifact.cache_digest)) is None + + def test_deleting_referenced_artifact_fails(testing_dag_bundle, session): artifact = _add_dags_and_artifact("dag_a", session=session) session.add(_make_handler(dag_id="dag_a", artifact=artifact)) From da3372e04cd3d092b2eacaca9d88c2849f7a4845 Mon Sep 17 00:00:00 2001 From: LIU ZHE YOU Date: Thu, 1 Oct 2026 07:12:27 +0000 Subject: [PATCH 23/25] Allow a NULL cache digest in ADR-0013's artifact table SQL The migration now lets an artifact store no digest, and later layers read this SQL as the spec. --- .../adr/lang-sdk/0013-persisted-task-handler-bindings.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airflow-core/adr/lang-sdk/0013-persisted-task-handler-bindings.md b/airflow-core/adr/lang-sdk/0013-persisted-task-handler-bindings.md index 74a93c0e4863c..d62927452aba0 100644 --- a/airflow-core/adr/lang-sdk/0013-persisted-task-handler-bindings.md +++ b/airflow-core/adr/lang-sdk/0013-persisted-task-handler-bindings.md @@ -162,7 +162,7 @@ CREATE TABLE lang_sdk_task_handler_artifact ( relative_fileloc VARCHAR(2000) NOT NULL, -- path within that bundle relative_fileloc_hash VARCHAR(32) NOT NULL, -- md5 of relative_fileloc; the path is too long to index size_bytes BIGINT NOT NULL, -- cheap fingerprint tier - cache_digest VARCHAR(128) NOT NULL, -- content fingerprint tier; see "The fast path" + cache_digest VARCHAR(128) NULL, -- content fingerprint tier, NULL if none is stored; see "The fast path" task_handlers JSON NOT NULL, -- {dag_id: [TaskHandlerDeclaration, ...]}, the probe answer last_probed_at TIMESTAMP NOT NULL, CONSTRAINT lang_sdk_task_handler_artifact_pkey PRIMARY KEY (id), From db4f8470d08d564ccf61b737220b610d6768c3ae Mon Sep 17 00:00:00 2001 From: ZHE YOU LIU Date: Tue, 29 Sep 2026 16:25:27 +0000 Subject: [PATCH 24/25] Move the Dag file processor's shared plumbing into a base class The native Dag parse and the task-handler probe start parse children other than the Python one, and they need the same parse log forwarding, request handling, readiness and cleanup. No behavior change. --- .../src/airflow/dag_processing/processor.py | 148 ++++++++++-------- 1 file changed, 82 insertions(+), 66 deletions(-) diff --git a/airflow-core/src/airflow/dag_processing/processor.py b/airflow-core/src/airflow/dag_processing/processor.py index 80e6d7161dec2..dbe6e5c2becb4 100644 --- a/airflow-core/src/airflow/dag_processing/processor.py +++ b/airflow-core/src/airflow/dag_processing/processor.py @@ -23,7 +23,7 @@ import traceback from collections.abc import Callable, Sequence from pathlib import Path -from typing import TYPE_CHECKING, Annotated, BinaryIO, ClassVar, Literal +from typing import TYPE_CHECKING, Annotated, Any, BinaryIO, ClassVar, Literal import attrs from pydantic import BaseModel, Field, TypeAdapter @@ -92,7 +92,7 @@ from airflow.sdk.definitions.context import Context from airflow.sdk.definitions.dag import DAG from airflow.sdk.definitions.mappedoperator import MappedOperator - from airflow.sdk.execution_time.supervisor import RequestHandler, RequestResult + from airflow.sdk.execution_time.supervisor import RequestHandler, RequestResult, ResponseSent from airflow.typing_compat import Self @@ -573,18 +573,17 @@ def in_process_api_server() -> InProcessExecutionAPI: @attrs.define(kw_only=True) -class DagFileProcessorProcess(WatchedSubprocess, LoggingMixin): +class BaseDagFileProcessorProcess(WatchedSubprocess, LoggingMixin): """ - Parses dags with Task SDK API. - - This class provides a wrapper and management around a subprocess to parse a specific DAG file. + Parse one Dag file in a child process for the Dag processor manager. - Since DAGs are written with the Task SDK, we need to parse them in a task SDK process such that - we can use the Task SDK definitions when serializing. This prevents potential conflicts with classes - in core Airflow. + The child's output goes to the file's parse log, and its requests are answered with + :attr:`client`. The parse is done once the child has exited and all its sockets are closed; + :attr:`parsing_result` then holds what it sent. Subclasses start the child and send it the + parse request. """ - logger_filehandle: BinaryIO + logger_filehandle: BinaryIO | None = None parsing_result: DagFileParsingResult | None = None decoder: ClassVar[TypeAdapter[ToManager]] = TypeAdapter[ToManager](ToManager) had_callbacks: bool = False # Track if this process was started with callbacks to prevent stale DAG detection false positives @@ -595,60 +594,6 @@ class DagFileProcessorProcess(WatchedSubprocess, LoggingMixin): bundle_name: str dag_file_rel_path: str - @classmethod - def start( # type: ignore[override] - cls, - *, - path: str | os.PathLike[str], - bundle_path: Path, - bundle_name: str, - dag_file_rel_path: str, - callbacks: list[CallbackRequest], - target: Callable[[], None] = _parse_file_entrypoint, - client: Client, - **kwargs, - ) -> Self: - logger = kwargs["logger"] - - # Parsing DAG files runs user code that can trigger macOS-unsafe ObjC - # initialization (secret backends, connection/variable lookups, HTTP - # clients). Fork+exec a clean interpreter there. Tests override `target` - # with a stub to exercise the base infrastructure; keep bare fork for those. - use_exec = target is _parse_file_entrypoint and supervisor._should_use_exec() - - # Pre-importing only helps the bare-fork child (it inherits the imports via - # copy-on-write). An exec'd child re-imports from scratch, so skip it there - # to avoid leaking user modules into the long-lived processor manager. - if not use_exec: - _pre_import_airflow_modules(os.fspath(path), logger) - - proc: Self = super().start( - target=target, - client=client, - bundle_name=bundle_name, - dag_file_rel_path=dag_file_rel_path, - use_exec=use_exec, - **kwargs, - ) - proc.had_callbacks = bool(callbacks) # Track if this process had callbacks - proc._on_child_started(callbacks, path, bundle_path, bundle_name) - return proc - - def _on_child_started( - self, - callbacks: list[CallbackRequest], - path: str | os.PathLike[str], - bundle_path: Path, - bundle_name: str, - ) -> None: - msg = DagFileParseRequest( - file=os.fspath(path), - bundle_path=bundle_path, - bundle_name=bundle_name, - callback_requests=callbacks, - ) - self.send_msg(msg, request_id=0) - def _get_target_loggers(self) -> tuple[FilteringBoundLogger, ...]: base = super()._get_target_loggers() if not self.subprocess_logs_to_stdout: @@ -674,11 +619,11 @@ def _create_log_forwarder( def _handle_parsing_result( self, msg: DagFileParsingResult, log: FilteringBoundLogger, req_id: int - ) -> RequestResult: + ) -> RequestResult | ResponseSent: self.parsing_result = msg return None, {} - _request_handlers: ClassVar[dict[type[BaseModel], RequestHandler[DagFileProcessorProcess]]] = { + _request_handlers: ClassVar[dict[type[BaseModel], RequestHandler[Any]]] = { **WatchedSubprocess._get_shared_request_handlers( DeleteVariable, GetConnection, @@ -720,6 +665,8 @@ def wait(self) -> int: def close(self): self.cleanup_sockets_after_kill() + if self.logger_filehandle is None: + return try: self.logger_filehandle.close() except OSError: @@ -728,3 +675,72 @@ def close(self): self.dag_file_rel_path, exc_info=True, ) + + +@attrs.define(kw_only=True) +class DagFileProcessorProcess(BaseDagFileProcessorProcess): + """ + Parses dags with Task SDK API. + + This class provides a wrapper and management around a subprocess to parse a specific DAG file. + + Since DAGs are written with the Task SDK, we need to parse them in a task SDK process such that + we can use the Task SDK definitions when serializing. This prevents potential conflicts with classes + in core Airflow. + """ + + logger_filehandle: BinaryIO + + @classmethod + def start( # type: ignore[override] + cls, + *, + path: str | os.PathLike[str], + bundle_path: Path, + bundle_name: str, + dag_file_rel_path: str, + callbacks: list[CallbackRequest], + target: Callable[[], None] = _parse_file_entrypoint, + client: Client, + **kwargs, + ) -> Self: + logger = kwargs["logger"] + + # Parsing DAG files runs user code that can trigger macOS-unsafe ObjC + # initialization (secret backends, connection/variable lookups, HTTP + # clients). Fork+exec a clean interpreter there. Tests override `target` + # with a stub to exercise the base infrastructure; keep bare fork for those. + use_exec = target is _parse_file_entrypoint and supervisor._should_use_exec() + + # Pre-importing only helps the bare-fork child (it inherits the imports via + # copy-on-write). An exec'd child re-imports from scratch, so skip it there + # to avoid leaking user modules into the long-lived processor manager. + if not use_exec: + _pre_import_airflow_modules(os.fspath(path), logger) + + proc: Self = super().start( + target=target, + client=client, + bundle_name=bundle_name, + dag_file_rel_path=dag_file_rel_path, + use_exec=use_exec, + **kwargs, + ) + proc.had_callbacks = bool(callbacks) # Track if this process had callbacks + proc._on_child_started(callbacks, path, bundle_path, bundle_name) + return proc + + def _on_child_started( + self, + callbacks: list[CallbackRequest], + path: str | os.PathLike[str], + bundle_path: Path, + bundle_name: str, + ) -> None: + msg = DagFileParseRequest( + file=os.fspath(path), + bundle_path=bundle_path, + bundle_name=bundle_name, + callback_requests=callbacks, + ) + self.send_msg(msg, request_id=0) From 7e26f0d11be3e98e894b51c3b2a0d2b864435210 Mon Sep 17 00:00:00 2001 From: LIU ZHE YOU Date: Thu, 1 Oct 2026 06:04:06 +0000 Subject: [PATCH 25/25] Test closing a Dag file processor that has no parse log file The base class makes logger_filehandle optional for parse children other than the Python one, but DagFileProcessorProcess still requires it, so nothing in this layer reached the close path without one. --- .../unit/dag_processing/test_processor.py | 22 ++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/airflow-core/tests/unit/dag_processing/test_processor.py b/airflow-core/tests/unit/dag_processing/test_processor.py index 0cb61aa02866a..9006876b477e9 100644 --- a/airflow-core/tests/unit/dag_processing/test_processor.py +++ b/airflow-core/tests/unit/dag_processing/test_processor.py @@ -27,7 +27,7 @@ import uuid import zipfile from collections.abc import Callable, Iterable -from socket import socketpair +from socket import socket, socketpair from typing import TYPE_CHECKING, Any, BinaryIO from unittest.mock import MagicMock, PropertyMock, patch @@ -54,6 +54,7 @@ from airflow.dag_processing.dagbag import DagBag from airflow.dag_processing.manager import DagFileProcessorManager, process_parse_results from airflow.dag_processing.processor import ( + BaseDagFileProcessorProcess, DagFileParseRequest, DagFileParsingResult, DagFileProcessorProcess, @@ -2546,6 +2547,25 @@ def test_handle_request_get_variable_masks_value_with_key(self, proc): } +class TestBaseDagFileProcessorProcess: + @patch.object(BaseDagFileProcessorProcess, "cleanup_sockets_after_kill", autospec=True) + def test_close_without_log_file_handle_cleans_up_sockets(self, cleanup_sockets_after_kill): + proc = BaseDagFileProcessorProcess( + process_log=structlog.get_logger(), + id=uuid.uuid4(), + pid=1234, + process=MagicMock(spec=supervisor.ProcessTracker), + stdin=MagicMock(spec=socket), + client=MagicMock(spec=Client), + bundle_name="mybundle", + dag_file_rel_path="dags/my_dag.py", + ) + + proc.close() + + cleanup_sockets_after_kill.assert_called_once_with(proc) + + class TestMultiTeamCallbackMetrics: """Tests for team_name tag on dag.callback_exceptions in multi-team mode."""