6666from airflow .sdk import SecretCache
6767from airflow .sdk .log import init_log_file , logging_processors
6868from airflow .typing_compat import assert_never
69- from airflow .utils .file import might_contain_dag
69+ from airflow .utils .file import list_py_file_paths , might_contain_dag
7070from airflow .utils .log .logging_mixin import LoggingMixin
7171from airflow .utils .net import get_hostname
7272from airflow .utils .process_utils import (
8888 from airflow .sdk .api .client import Client
8989
9090
91- log = logging .getLogger (__name__ )
92-
93-
9491class DagParsingStat (NamedTuple ):
9592 """Information on processing progress."""
9693
@@ -161,62 +158,6 @@ def utc_epoch() -> datetime:
161158 return result
162159
163160
164- def discover_dag_file_paths (
165- directory : str | os .PathLike [str ] | None ,
166- bundle_name : str = "" ,
167- safe_mode : bool = conf .getboolean ("core" , "DAG_DISCOVERY_SAFE_MODE" , fallback = True ),
168- ) -> list [str ]:
169- """
170- Discover paths of DAG files within a directory.
171-
172- Walks ``directory`` (honouring ``.airflowignore``) and returns each file that is
173- either a Python DAG candidate (``.py`` source or ZIP archive that passes
174- :func:`~airflow.utils.file.might_contain_dag`) or accepted by a registered coordinator's
175- :meth:`~airflow.sdk.execution_time.coordinator.BaseCoordinator.can_handle_dag_file`
176- (e.g. a ``.jar`` for the Java SDK, a self-contained executable for the Go SDK).
177-
178- Coordinator handling takes precedence over the generic ZIP heuristic so that,
179- for example, a ``.jar`` is delegated to its coordinator rather than being
180- scanned for embedded ``.py`` modules.
181-
182- :param directory: Directory to scan, or a single file path. ``None`` returns
183- an empty list. A single file is returned as-is without filtering.
184- :param bundle_name: Bundle name forwarded to ``can_handle_dag_file``.
185- :param safe_mode: Whether to apply the Python DAG heuristic; see
186- :func:`~airflow.utils.file.might_contain_dag`.
187- :return: Absolute paths discovered as DAG sources.
188- """
189- if directory is None :
190- return []
191- if os .path .isfile (directory ):
192- return [str (directory )]
193- if not os .path .isdir (directory ):
194- return []
195-
196- from airflow ._shared .module_loading .file_discovery import find_path_from_directory
197- from airflow .providers_manager import ProvidersManager
198-
199- coordinators = ProvidersManager ().coordinators
200- ignore_file_syntax = conf .get_mandatory_value ("core" , "DAG_IGNORE_FILE_SYNTAX" , fallback = "glob" )
201-
202- file_paths : list [str ] = []
203- for file_path in find_path_from_directory (directory , ".airflowignore" , ignore_file_syntax ):
204- path = Path (file_path )
205- try :
206- if not path .is_file ():
207- continue
208- if path .suffix == ".py" :
209- if might_contain_dag (file_path , safe_mode ):
210- file_paths .append (file_path )
211- elif any (c .can_handle_dag_file (bundle_name , file_path ) for c in coordinators ):
212- file_paths .append (file_path )
213- elif zipfile .is_zipfile (path ) and might_contain_dag (file_path , safe_mode ):
214- file_paths .append (file_path )
215- except Exception :
216- log .exception ("Error while examining %s" , file_path )
217- return file_paths
218-
219-
220161class _StubSelector (selectors .BaseSelector ):
221162 """
222163 Stub to stand in until the real selector is created.
@@ -889,11 +830,9 @@ def _refresh_dag_bundles(self, known_files: dict[str, set[DagFileInfo]]):
889830
890831 def _find_files_in_bundle (self , bundle : BaseDagBundle ) -> list [Path ]:
891832 """Get relative paths for dag files from bundle dir."""
833+ # Build up a list of Python files that could contain DAGs
892834 self .log .info ("Searching for files in %s at %s" , bundle .name , bundle .path )
893- rel_paths = [
894- Path (x ).relative_to (bundle .path )
895- for x in discover_dag_file_paths (bundle .path , bundle_name = bundle .name )
896- ]
835+ rel_paths = [Path (x ).relative_to (bundle .path ) for x in list_py_file_paths (bundle .path )]
897836 self .log .info ("Found %s files for bundle %s" , len (rel_paths ), bundle .name )
898837
899838 return rel_paths
@@ -905,13 +844,7 @@ def _get_observed_filelocs(self, present: set[DagFileInfo]) -> set[str]:
905844 For regular files this includes the relative file path.
906845 For ZIP archives this includes DAG-like inner paths such as
907846 ``archive.zip/dag.py``.
908-
909- Files claimed by a registered runtime coordinator (e.g. ``.jar``)
910- are treated as opaque files rather than ZIP archives.
911847 """
912- from airflow .providers_manager import ProvidersManager
913-
914- coordinators = ProvidersManager ().coordinators
915848
916849 def find_zipped_dags (abs_path : os .PathLike ) -> Iterator [str ]:
917850 """Yield absolute paths for DAG-like files inside a ZIP archive."""
@@ -926,10 +859,7 @@ def find_zipped_dags(abs_path: os.PathLike) -> Iterator[str]:
926859 observed_filelocs : set [str ] = set ()
927860 for info in present :
928861 abs_path = str (info .absolute_path )
929- handled_by_coordinator = any (
930- c .can_handle_dag_file (info .bundle_name , abs_path ) for c in coordinators
931- )
932- if abs_path .endswith (".py" ) or handled_by_coordinator or not zipfile .is_zipfile (abs_path ):
862+ if abs_path .endswith (".py" ) or not zipfile .is_zipfile (abs_path ):
933863 observed_filelocs .add (str (info .rel_path ))
934864 else :
935865 if TYPE_CHECKING :
0 commit comments