From 4d7a3b7c8ec8f216b1d9712eebf6a6ab61496d53 Mon Sep 17 00:00:00 2001 From: tnazarew Date: Wed, 21 Oct 2026 16:54:10 +0200 Subject: [PATCH] spark bigquery: drain listener bus before stopping Spark The script exited right after the write, so the SparkContext could stop while the OpenLineage listener was still handling the write's SQLExecutionEnd. The BigQuery connector's lineage extension then fails with "Cannot call methods on a stopped SparkContext" and the COMPLETE event is emitted with empty outputs (issues #339, #341; Spark 3.5.1, OpenLineage 1.54.0, intermittent). Wait for the listener bus to drain, then stop Spark explicitly. Co-Authored-By: Claude Opus 5.5 Signed-off-by: tnazarew --- producer/spark_dataproc/scenarios/bigquery/test/test.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/producer/spark_dataproc/scenarios/bigquery/test/test.py b/producer/spark_dataproc/scenarios/bigquery/test/test.py index 700c31f1..8c087a30 100644 --- a/producer/spark_dataproc/scenarios/bigquery/test/test.py +++ b/producer/spark_dataproc/scenarios/bigquery/test/test.py @@ -24,3 +24,9 @@ .option("temporaryGcsBucket", "open-lineage-e2e") \ .mode('overwrite') \ .save() + +# Let the OpenLineage listener handle the write's end event while the SparkContext +# is still alive. Otherwise the BigQuery lineage extension can fail on a stopped +# context and the COMPLETE event is emitted without outputs. +spark.sparkContext._jsc.sc().listenerBus().waitUntilEmpty(60000) +spark.stop()