Skip to content

Commit 7d57137

Browse files
committed
Move executable coordinator back to SDK
1 parent 48c767b commit 7d57137

36 files changed

Lines changed: 330 additions & 1431 deletions

File tree

‎.github/boring-cyborg.yml‎

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -261,9 +261,6 @@ labelPRBasedOnFilePath:
261261
provider:samba:
262262
- providers/samba/**
263263

264-
provider:sdk-executable:
265-
- providers/sdk/executable/**
266-
267264
provider:segment:
268265
- providers/segment/**
269266

‎airflow-core/src/airflow/dag_processing/manager.py‎

Lines changed: 4 additions & 74 deletions
Original file line numberDiff line numberDiff line change
@@ -66,7 +66,7 @@
6666
from airflow.sdk import SecretCache
6767
from airflow.sdk.log import init_log_file, logging_processors
6868
from airflow.typing_compat import assert_never
69-
from airflow.utils.file import might_contain_dag
69+
from airflow.utils.file import list_py_file_paths, might_contain_dag
7070
from airflow.utils.log.logging_mixin import LoggingMixin
7171
from airflow.utils.net import get_hostname
7272
from airflow.utils.process_utils import (
@@ -88,9 +88,6 @@
8888
from airflow.sdk.api.client import Client
8989

9090

91-
log = logging.getLogger(__name__)
92-
93-
9491
class DagParsingStat(NamedTuple):
9592
"""Information on processing progress."""
9693

@@ -161,62 +158,6 @@ def utc_epoch() -> datetime:
161158
return result
162159

163160

164-
def discover_dag_file_paths(
165-
directory: str | os.PathLike[str] | None,
166-
bundle_name: str = "",
167-
safe_mode: bool = conf.getboolean("core", "DAG_DISCOVERY_SAFE_MODE", fallback=True),
168-
) -> list[str]:
169-
"""
170-
Discover paths of DAG files within a directory.
171-
172-
Walks ``directory`` (honouring ``.airflowignore``) and returns each file that is
173-
either a Python DAG candidate (``.py`` source or ZIP archive that passes
174-
:func:`~airflow.utils.file.might_contain_dag`) or accepted by a registered coordinator's
175-
:meth:`~airflow.sdk.execution_time.coordinator.BaseCoordinator.can_handle_dag_file`
176-
(e.g. a ``.jar`` for the Java SDK, a self-contained executable for the Go SDK).
177-
178-
Coordinator handling takes precedence over the generic ZIP heuristic so that,
179-
for example, a ``.jar`` is delegated to its coordinator rather than being
180-
scanned for embedded ``.py`` modules.
181-
182-
:param directory: Directory to scan, or a single file path. ``None`` returns
183-
an empty list. A single file is returned as-is without filtering.
184-
:param bundle_name: Bundle name forwarded to ``can_handle_dag_file``.
185-
:param safe_mode: Whether to apply the Python DAG heuristic; see
186-
:func:`~airflow.utils.file.might_contain_dag`.
187-
:return: Absolute paths discovered as DAG sources.
188-
"""
189-
if directory is None:
190-
return []
191-
if os.path.isfile(directory):
192-
return [str(directory)]
193-
if not os.path.isdir(directory):
194-
return []
195-
196-
from airflow._shared.module_loading.file_discovery import find_path_from_directory
197-
from airflow.providers_manager import ProvidersManager
198-
199-
coordinators = ProvidersManager().coordinators
200-
ignore_file_syntax = conf.get_mandatory_value("core", "DAG_IGNORE_FILE_SYNTAX", fallback="glob")
201-
202-
file_paths: list[str] = []
203-
for file_path in find_path_from_directory(directory, ".airflowignore", ignore_file_syntax):
204-
path = Path(file_path)
205-
try:
206-
if not path.is_file():
207-
continue
208-
if path.suffix == ".py":
209-
if might_contain_dag(file_path, safe_mode):
210-
file_paths.append(file_path)
211-
elif any(c.can_handle_dag_file(bundle_name, file_path) for c in coordinators):
212-
file_paths.append(file_path)
213-
elif zipfile.is_zipfile(path) and might_contain_dag(file_path, safe_mode):
214-
file_paths.append(file_path)
215-
except Exception:
216-
log.exception("Error while examining %s", file_path)
217-
return file_paths
218-
219-
220161
class _StubSelector(selectors.BaseSelector):
221162
"""
222163
Stub to stand in until the real selector is created.
@@ -889,11 +830,9 @@ def _refresh_dag_bundles(self, known_files: dict[str, set[DagFileInfo]]):
889830

890831
def _find_files_in_bundle(self, bundle: BaseDagBundle) -> list[Path]:
891832
"""Get relative paths for dag files from bundle dir."""
833+
# Build up a list of Python files that could contain DAGs
892834
self.log.info("Searching for files in %s at %s", bundle.name, bundle.path)
893-
rel_paths = [
894-
Path(x).relative_to(bundle.path)
895-
for x in discover_dag_file_paths(bundle.path, bundle_name=bundle.name)
896-
]
835+
rel_paths = [Path(x).relative_to(bundle.path) for x in list_py_file_paths(bundle.path)]
897836
self.log.info("Found %s files for bundle %s", len(rel_paths), bundle.name)
898837

899838
return rel_paths
@@ -905,13 +844,7 @@ def _get_observed_filelocs(self, present: set[DagFileInfo]) -> set[str]:
905844
For regular files this includes the relative file path.
906845
For ZIP archives this includes DAG-like inner paths such as
907846
``archive.zip/dag.py``.
908-
909-
Files claimed by a registered runtime coordinator (e.g. ``.jar``)
910-
are treated as opaque files rather than ZIP archives.
911847
"""
912-
from airflow.providers_manager import ProvidersManager
913-
914-
coordinators = ProvidersManager().coordinators
915848

916849
def find_zipped_dags(abs_path: os.PathLike) -> Iterator[str]:
917850
"""Yield absolute paths for DAG-like files inside a ZIP archive."""
@@ -926,10 +859,7 @@ def find_zipped_dags(abs_path: os.PathLike) -> Iterator[str]:
926859
observed_filelocs: set[str] = set()
927860
for info in present:
928861
abs_path = str(info.absolute_path)
929-
handled_by_coordinator = any(
930-
c.can_handle_dag_file(info.bundle_name, abs_path) for c in coordinators
931-
)
932-
if abs_path.endswith(".py") or handled_by_coordinator or not zipfile.is_zipfile(abs_path):
862+
if abs_path.endswith(".py") or not zipfile.is_zipfile(abs_path):
933863
observed_filelocs.add(str(info.rel_path))
934864
else:
935865
if TYPE_CHECKING:

0 commit comments

Comments
 (0)