diff --git a/producer/spark_dataproc/runner/get_openlineage_jar.sh b/producer/spark_dataproc/runner/get_openlineage_jar.sh index bf418457..23adfacf 100755 --- a/producer/spark_dataproc/runner/get_openlineage_jar.sh +++ b/producer/spark_dataproc/runner/get_openlineage_jar.sh @@ -5,23 +5,19 @@ set -euxo pipefail readonly VM_SPARK_JARS_DIR=/usr/lib/spark/jars -readonly SPARK_BQ_CONNECTOR_URL=$(/usr/share/google/get_metadata_value attributes/SPARK_BQ_CONNECTOR_URL || echo "") -readonly OPENLINEAGE_SPARK_URL=$(/usr/share/google/get_metadata_value attributes/OPENLINEAGE_SPARK_URL || echo "") -readonly SPARK_SPANNER_CONNECTOR_URL=$(/usr/share/google/get_metadata_value attributes/SPARK_SPANNER_CONNECTOR_URL || echo "") -readonly SPARK_BIGTABLE_CONNECTOR_URL=$(/usr/share/google/get_metadata_value attributes/SPARK_BIGTABLE_CONNECTOR_URL || echo "") readonly SKIP_ICEBERG_AND_DELTA=$(/usr/share/google/get_metadata_value attributes/SKIP_ICEBERG_AND_DELTA || echo "") -if [[ -n "${OPENLINEAGE_SPARK_URL}" ]]; then - bq_url="${SPARK_BQ_CONNECTOR_URL}" - ol_url="${OPENLINEAGE_SPARK_URL}" - spanner_url="${SPARK_SPANNER_CONNECTOR_URL}" - bigtable_url="${SPARK_BIGTABLE_CONNECTOR_URL}" -else - bq_url="gs://open-lineage-e2e/jars/spark-3.5-bigquery-0.42.4.jar" - ol_url="gs://open-lineage-e2e/jars/openlineage-spark_2.12-1.29.0-SNAPSHOT.jar" - spanner_url="gs://open-lineage-e2e/jars/spark-3.1-spanner-1.1.0.jar" - bigtable_url="gs://open-lineage-e2e/jars/spark-bigtable_2.12-0.3.0.jar" -fi +# Returns the metadata attribute value, or the given default when the attribute is not set. +metadata_or_default() { + local value + value=$(/usr/share/google/get_metadata_value "attributes/$1" || echo "") + echo "${value:-$2}" +} + +bq_url=$(metadata_or_default SPARK_BQ_CONNECTOR_URL "gs://open-lineage-e2e/jars/spark-3.5-bigquery-0.42.4.jar") +ol_url=$(metadata_or_default OPENLINEAGE_SPARK_URL "gs://open-lineage-e2e/jars/openlineage-spark_2.12-1.29.0-SNAPSHOT.jar") +spanner_url=$(metadata_or_default SPARK_SPANNER_CONNECTOR_URL "gs://open-lineage-e2e/jars/spark-3.1-spanner-1.1.0.jar") +bigtable_url=$(metadata_or_default SPARK_BIGTABLE_CONNECTOR_URL "gs://open-lineage-e2e/jars/spark-bigtable_2.12-0.3.0.jar") postgresql_url="gs://open-lineage-e2e/jars/postgresql-42.5.6.jar" delta_spark_url="gs://open-lineage-e2e/jars/delta-spark_2.12-3.3.1.jar"