diff --git a/producer/spark_dataproc/scenarios/bigquery/test/test.py b/producer/spark_dataproc/scenarios/bigquery/test/test.py index 700c31f1..8c087a30 100644 --- a/producer/spark_dataproc/scenarios/bigquery/test/test.py +++ b/producer/spark_dataproc/scenarios/bigquery/test/test.py @@ -24,3 +24,9 @@ .option("temporaryGcsBucket", "open-lineage-e2e") \ .mode('overwrite') \ .save() + +# Let the OpenLineage listener handle the write's end event while the SparkContext +# is still alive. Otherwise the BigQuery lineage extension can fail on a stopped +# context and the COMPLETE event is emitted without outputs. +spark.sparkContext._jsc.sc().listenerBus().waitUntilEmpty(60000) +spark.stop()