Place dependent k8s env vars after other env vars

task_processing/plugins/kubernetes/utils.py

@@ -1,5 +1,6 @@
 import hashlib
 import logging
+import re
 from typing import Dict
 from typing import List
 from typing import Optional
@@ -44,6 +45,9 @@
 logger = logging.getLogger(__name__)
 
 DEFAULT_PROJECTED_SA_TOKEN_EXPIRATION_SECONDS = 1800
+# Kubernetes only expands $(VAR_NAME) references to vars defined earlier in the env list
+# ($$(VAR_NAME) is an escaped literal, so we don't treat it as a reference)
+DEPENDENT_ENV_VAR_RE = re.compile(r"(?<!\$)\$\([A-Za-z_][A-Za-z0-9_]*\)")
 
 
 def get_capabilities_for_capability_changes(
@@ -86,12 +90,20 @@ def get_kubernetes_env_vars(
     Given a dict of environment variables, transform them into the corresponding Kubernetes
     representation. This function will replace any secret placeholders with the value of
     the actual secret.
+
+    Variables that reference other variables (e.g. "$(PAASTA_POD_IP):8080") are placed at
+    the end so that Kubernetes can expand them.
     """
-    env_vars = [
-        V1EnvVar(name=key, value=value)
-        for key, value in environment.items()
-        if key not in secret_environment.keys()
-    ]
+    env_vars = []
+    dependent_env_vars = []
+    for key, value in environment.items():
+        if key in secret_environment.keys():
+            continue
+        env_var = V1EnvVar(name=key, value=value)
+        if DEPENDENT_ENV_VAR_RE.search(value):
+            dependent_env_vars.append(env_var)
+        else:
+            env_vars.append(env_var)
 
     secret_env_vars = [
         V1EnvVar(
@@ -119,7 +131,7 @@ def get_kubernetes_env_vars(
         for key, value in field_selector_environment.items()
     ]
 
-    return env_vars + secret_env_vars + field_selector_env_vars
+    return env_vars + secret_env_vars + field_selector_env_vars + dependent_env_vars
 
 
 def get_sanitised_kubernetes_name(

tests/unit/plugins/kubernetes/kubernetes_utils_test.py

@@ -455,6 +455,39 @@ def test_get_kubernetes_env_vars():
     )
 
 
+def test_get_kubernetes_env_vars_dependent_vars_last():
+    env_vars = get_kubernetes_env_vars(
+        environment=pmap(
+            {
+                "POD_ADDR": "$(PAASTA_POD_IP):8080",
+                "FAKE_PLAIN_VAR": "not_secret_data",
+                "ESCAPED_VAR": "$$(NOT_A_REFERENCE)",
+                "SECRET_URL": "https://user:$(FAKE_SECRET)@host",
+            }
+        ),
+        secret_environment=pmap(
+            {
+                "FAKE_SECRET": {
+                    "secret_name": "taskns-secret-taskname-some--secret--name",
+                    "key": "some_secret_name",
+                },
+            }
+        ),
+        field_selector_environment=pmap(
+            {"PAASTA_POD_IP": {"field_path": "status.podIP"}}
+        ),
+    )
+
+    names = [env_var.name for env_var in env_vars]
+    assert set(names[:4]) == {
+        "FAKE_PLAIN_VAR",
+        "ESCAPED_VAR",
+        "FAKE_SECRET",
+        "PAASTA_POD_IP",
+    }
+    assert set(names[4:]) == {"POD_ADDR", "SECRET_URL"}
+
+
 def test_get_node_affinity_ok():
     affinities = pvector(
         [