Skip to content

Reduce per-task DB round trips in the asset dependency graph API #71915

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
vatsrahul1001 merged 2 commits into from
Sep 7, 2026
Merged

Reduce per-task DB round trips in the asset dependency graph API #71915

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
1ee5a1d
Reduce per-task DB round trips in the asset dependency graph API
ColtenOuO Aug 20, 2026
34a8c84
Add outlet-resolution batching test for the asset dependency graph
ColtenOuO Aug 29, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
56 changes: 29 additions & 27 deletions airflow-core/src/airflow/api_fastapi/core_api/services/ui/dependencies.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -394,7 +394,7 @@ def get_data_dependencies(
asset_id: int, session: Session, readable_dag_ids: set[str] | None = None
) -> dict[str, list[dict]]:
"""Get full task dependencies for an asset."""
from sqlalchemy import select, union_all
from sqlalchemy import select, tuple_, union_all
from sqlalchemy.orm import selectinload

from airflow.models.asset import (
Expand Down Expand Up @@ -458,6 +458,8 @@ def get_data_dependencies(
next_frontier: set[int] = set()
pending_dag_ids: set[str] = set()
triggering_asset_node_ids_by_dag: dict[str, set[str]] = defaultdict(set)
pending_inlet_keys: set[tuple[str, str]] = set()
pending_outlet_keys: set[tuple[str, str]] = set()

for asset in assets:
asset_node_id = f"asset:{asset.id}"
Expand Down Expand Up @@ -491,15 +493,7 @@ def get_data_dependencies(
# Find other assets this task consumes (inlets) to trace upstream
if task_key not in processed_tasks:
processed_tasks.add(task_key)
inlet_refs = session.scalars(
select(TaskInletAssetReference).where(
TaskInletAssetReference.dag_id == ref.dag_id,
TaskInletAssetReference.task_id == ref.task_id,
)
).all()
for inlet_ref in inlet_refs:
if inlet_ref.asset_id not in processed_assets:
next_frontier.add(inlet_ref.asset_id)
pending_inlet_keys.add(task_key)

# Process consuming tasks (tasks that input this asset)
for ref in asset.consuming_tasks:
Expand All @@ -523,15 +517,7 @@ def get_data_dependencies(
# Find other assets this task produces (outlets) to trace downstream
if task_key not in processed_tasks:
processed_tasks.add(task_key)
outlet_refs = session.scalars(
select(TaskOutletAssetReference).where(
TaskOutletAssetReference.dag_id == ref.dag_id,
TaskOutletAssetReference.task_id == ref.task_id,
)
).all()
for outlet_ref in outlet_refs:
if outlet_ref.asset_id not in processed_assets:
next_frontier.add(outlet_ref.asset_id)
pending_outlet_keys.add(task_key)

# Process Dags scheduled by this asset at the Dag level (`schedule=...`). These have
# no task-level inlet reference and would otherwise be a dead end in this graph --
Expand Down Expand Up @@ -599,15 +585,31 @@ def get_data_dependencies(
# Find other assets this entry task produces (outlets) to trace downstream.
if task_key not in processed_tasks:
processed_tasks.add(task_key)
outlet_refs = session.scalars(
select(TaskOutletAssetReference).where(
TaskOutletAssetReference.dag_id == dag_id,
TaskOutletAssetReference.task_id == entry_task_id,
pending_outlet_keys.add(task_key)

if pending_inlet_keys:
inlet_refs = session.scalars(
select(TaskInletAssetReference).where(
tuple_(TaskInletAssetReference.dag_id, TaskInletAssetReference.task_id).in_(
pending_inlet_keys
)
)
).all()
for inlet_ref in inlet_refs:
if inlet_ref.asset_id not in processed_assets:
next_frontier.add(inlet_ref.asset_id)

if pending_outlet_keys:
outlet_refs = session.scalars(
select(TaskOutletAssetReference).where(
tuple_(TaskOutletAssetReference.dag_id, TaskOutletAssetReference.task_id).in_(
pending_outlet_keys
)
).all()
for outlet_ref in outlet_refs:
if outlet_ref.asset_id not in processed_assets:
next_frontier.add(outlet_ref.asset_id)
)
).all()
for outlet_ref in outlet_refs:
if outlet_ref.asset_id not in processed_assets:
next_frontier.add(outlet_ref.asset_id)
Comment thread
ColtenOuO marked this conversation as resolved.

frontier = next_frontier

Expand Down
72 changes: 72 additions & 0 deletions airflow-core/tests/unit/api_fastapi/core_api/routes/ui/test_dependencies.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -868,6 +868,78 @@ def test_data_dependencies_batches_entry_point_resolution_across_scheduled_dags(
entry_task_node_id = f"task:{dag_id}__SEPARATOR__entry_task"
assert nodes_by_id[entry_task_node_id]["type"] == "task"

def test_data_dependencies_batches_inlet_resolution_across_producing_tasks(
self, dag_maker, test_client, session
):
"""Producing tasks trace upstream through their own inlets; that inlet lookup must be
batched across all producing tasks in a round, not issued once per task."""
asset_a = Asset(uri="s3://data-dep-bucket/inlet-batch-a", name="data_dep_inlet_batch_asset_a")
dag_ids = [f"data_dep_inlet_batch_producer_{i}" for i in range(3)]

for dag_id in dag_ids:
with dag_maker(
dag_id=dag_id,
schedule=None,
serialized=True,
session=session,
):
EmptyOperator(task_id="produce", outlets=[asset_a])

dag_maker.sync_dagbag_to_db()

asset_a_id = session.scalar(
select(AssetModel.id).where(AssetModel.name == "data_dep_inlet_batch_asset_a")
)

expected_query_count = 9
with assert_queries_count(expected_query_count):
response = test_client.get(
"/dependencies", params={"node_id": f"asset:{asset_a_id}", "dependency_type": "data"}
)
assert response.status_code == 200

result = response.json()
nodes_by_id = {node["id"]: node for node in result["nodes"]}
for dag_id in dag_ids:
task_node_id = f"task:{dag_id}__SEPARATOR__produce"
assert nodes_by_id[task_node_id]["type"] == "task"

def test_data_dependencies_batches_outlet_resolution_across_consuming_tasks(
self, dag_maker, test_client, session
):
"""Consuming tasks trace downstream through their own outlets; that outlet lookup must be
batched across all consuming tasks in a round, not issued once per task."""
asset_a = Asset(uri="s3://data-dep-bucket/outlet-batch-a", name="data_dep_outlet_batch_asset_a")
dag_ids = [f"data_dep_outlet_batch_consumer_{i}" for i in range(3)]

for dag_id in dag_ids:
with dag_maker(
dag_id=dag_id,
schedule=None,
serialized=True,
session=session,
):
EmptyOperator(task_id="consume", inlets=[asset_a])

dag_maker.sync_dagbag_to_db()

asset_a_id = session.scalar(
select(AssetModel.id).where(AssetModel.name == "data_dep_outlet_batch_asset_a")
)

expected_query_count = 9
with assert_queries_count(expected_query_count):
response = test_client.get(
"/dependencies", params={"node_id": f"asset:{asset_a_id}", "dependency_type": "data"}
)
assert response.status_code == 200

result = response.json()
nodes_by_id = {node["id"]: node for node in result["nodes"]}
for dag_id in dag_ids:
task_node_id = f"task:{dag_id}__SEPARATOR__consume"
assert nodes_by_id[task_node_id]["type"] == "task"

def test_data_dependencies_attributes_alias_produced_asset_to_source_task(
self, dag_maker, test_client, session
):
Expand Down