From 0f66f4e899a21ec334aee9718ebcb76bc8e8dd6b Mon Sep 17 00:00:00 2001 From: morazow Date: Mon, 26 Oct 2026 16:09:52 +0100 Subject: [PATCH] [helm] Publish each listener's advertised address as a pod env var Server pods now get one environment variable per listener, FLUSS_ADVERTISED_LISTENER_, holding that listener's advertised address as :. Tools such as the Fluss Agent can read these values instead of rebuilding the addresses outside the chart. Before this change, the startup command built bind.listeners and advertised.listeners as shell variables and wrote them into server.yaml. Only CLIENT was advertised. Fluss advertises a listener that is missing from advertised.listeners at its bind address, so INTERNAL was advertised as :. The chart now: - adds the env entries after POD_NAME, POD_NAMESPACE and POD_IP and references them with $(VAR), so Kubernetes expands the values when it starts the container - publishes INTERNAL as $(POD_IP):, the address Fluss already registers, and every other listener as the pod's headless-service DNS name - builds advertised.listeners from the new env vars, so the published and the registered addresses cannot differ - generates the env vars, bind.listeners and advertised.listeners by looping over .Values.listeners, so additional listeners need no template changes Fluss behaves the same as before: the registered listeners are the same as with the previous chart. Tested with new helm-unittest cases (listeners_env_test.yaml) and on a kind cluster with apache/fluss:1.0.0-rc3. On every pod, the env values matched the listeners the server registered in ZooKeeper. --- helm/templates/_helpers.tpl | 43 +++++++ helm/templates/sts-coordinator.yaml | 5 +- helm/templates/sts-tablet.yaml | 5 +- helm/tests/listeners_env_test.yaml | 111 ++++++++++++++++++ .../install-deploy/deploying-with-helm.md | 7 ++ 5 files changed, 167 insertions(+), 4 deletions(-) create mode 100644 helm/tests/listeners_env_test.yaml diff --git a/helm/templates/_helpers.tpl b/helm/templates/_helpers.tpl index 3933debca68..b58ec9beeae 100644 --- a/helm/templates/_helpers.tpl +++ b/helm/templates/_helpers.tpl @@ -131,3 +131,46 @@ imagePullSecrets: {{- end -}} {{- end -}} + +{{/* +Env entries publishing each listener's advertised address as +FLUSS_ADVERTISED_LISTENER_=:, one per entry in .Values.listeners. +Fluss advertises a listener missing from advertised.listeners at its bind address, +so INTERNAL is published as the pod IP; the others use the pod's headless-service DNS name. +The values reference POD_* via $(VAR), so they must be rendered after those entries. +Usage: include "fluss.listeners.advertisedEnv" (dict "root" . "service" "coordinator-server-hs") +*/}} +{{- define "fluss.listeners.advertisedEnv" -}} +{{- $service := .service -}} +{{- range $name, $listener := .root.Values.listeners }} +- name: FLUSS_ADVERTISED_LISTENER_{{ upper $name }} + {{- if eq $name "internal" }} + value: $(POD_IP):{{ $listener.port }} + {{- else }} + value: $(POD_NAME).{{ $service }}.$(POD_NAMESPACE).svc.cluster.local:{{ $listener.port }} + {{- end }} +{{- end }} +{{- end -}} + +{{/* +bind.listeners value: every listener bound on the pod IP. +*/}} +{{- define "fluss.listeners.bind" -}} +{{- $out := list -}} +{{- range $name, $listener := .Values.listeners -}} +{{- $out = append $out (printf "%s://${POD_IP}:%v" (upper $name) $listener.port) -}} +{{- end -}} +{{- join ", " $out -}} +{{- end -}} + +{{/* +advertised.listeners value, read from the env entries of "fluss.listeners.advertisedEnv" +so the published addresses and the configured ones cannot differ. +*/}} +{{- define "fluss.listeners.advertised" -}} +{{- $out := list -}} +{{- range $name, $_ := .Values.listeners -}} +{{- $out = append $out (printf "%s://${FLUSS_ADVERTISED_LISTENER_%s}" (upper $name) (upper $name)) -}} +{{- end -}} +{{- join ", " $out -}} +{{- end -}} diff --git a/helm/templates/sts-coordinator.yaml b/helm/templates/sts-coordinator.yaml index 583cfcd9cbd..ae530b80bc5 100644 --- a/helm/templates/sts-coordinator.yaml +++ b/helm/templates/sts-coordinator.yaml @@ -94,6 +94,7 @@ spec: valueFrom: fieldRef: fieldPath: metadata.namespace + {{- include "fluss.listeners.advertisedEnv" (dict "root" . "service" "coordinator-server-hs") | trim | nindent 12 }} - name: NODE_IP valueFrom: fieldRef: @@ -124,8 +125,8 @@ spec: export FLUSS_SERVER_ID=${POD_NAME##*-} && \ cp /opt/conf/server.yaml $FLUSS_HOME/conf && \ - BIND_LISTENERS="INTERNAL://${POD_IP}:{{ .Values.listeners.internal.port }}, CLIENT://${POD_IP}:{{ .Values.listeners.client.port }}" && \ - ADVERTISED_LISTENERS="CLIENT://${POD_NAME}.coordinator-server-hs.${POD_NAMESPACE}.svc.cluster.local:{{ .Values.listeners.client.port }}" && \ + BIND_LISTENERS="{{ include "fluss.listeners.bind" . }}" && \ + ADVERTISED_LISTENERS="{{ include "fluss.listeners.advertised" . }}" && \ echo "" >> $FLUSS_HOME/conf/server.yaml && \ echo "bind.listeners: ${BIND_LISTENERS}" >> $FLUSS_HOME/conf/server.yaml && \ diff --git a/helm/templates/sts-tablet.yaml b/helm/templates/sts-tablet.yaml index 5407b2a6a95..8776ffef322 100644 --- a/helm/templates/sts-tablet.yaml +++ b/helm/templates/sts-tablet.yaml @@ -94,6 +94,7 @@ spec: valueFrom: fieldRef: fieldPath: metadata.namespace + {{- include "fluss.listeners.advertisedEnv" (dict "root" . "service" "tablet-server-hs") | trim | nindent 12 }} {{- if (include "fluss.security.jaas.required" .) }} - name: FLUSS_ENV_JAVA_OPTS value: "-Djava.security.auth.login.config=/etc/fluss/conf/jaas.conf" @@ -120,8 +121,8 @@ spec: export FLUSS_SERVER_ID=${POD_NAME##*-} && \ cp /opt/conf/server.yaml $FLUSS_HOME/conf && \ - BIND_LISTENERS="INTERNAL://${POD_IP}:{{ .Values.listeners.internal.port }}, CLIENT://${POD_IP}:{{ .Values.listeners.client.port }}" && \ - ADVERTISED_LISTENERS="CLIENT://${POD_NAME}.tablet-server-hs.${POD_NAMESPACE}.svc.cluster.local:{{ .Values.listeners.client.port }}" && \ + BIND_LISTENERS="{{ include "fluss.listeners.bind" . }}" && \ + ADVERTISED_LISTENERS="{{ include "fluss.listeners.advertised" . }}" && \ echo "" >> $FLUSS_HOME/conf/server.yaml && \ echo "tablet-server.id: ${FLUSS_SERVER_ID}" >> $FLUSS_HOME/conf/server.yaml && \ diff --git a/helm/tests/listeners_env_test.yaml b/helm/tests/listeners_env_test.yaml new file mode 100644 index 00000000000..68d0d81b9f0 --- /dev/null +++ b/helm/tests/listeners_env_test.yaml @@ -0,0 +1,111 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +suite: coordinator advertised listener env +templates: + - templates/sts-coordinator.yaml + +tests: + - it: should publish the CLIENT advertised address as an env var + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: FLUSS_ADVERTISED_LISTENER_CLIENT + value: $(POD_NAME).coordinator-server-hs.$(POD_NAMESPACE).svc.cluster.local:9124 + + - it: should publish the INTERNAL advertised address as the pod IP + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: FLUSS_ADVERTISED_LISTENER_INTERNAL + value: $(POD_IP):9123 + + - it: should build advertised.listeners from the published env vars + asserts: + - matchRegex: + path: spec.template.spec.containers[0].command[2] + pattern: 'ADVERTISED_LISTENERS="CLIENT://\$\{FLUSS_ADVERTISED_LISTENER_CLIENT\}, INTERNAL://\$\{FLUSS_ADVERTISED_LISTENER_INTERNAL\}"' + + - it: should bind every listener on the pod IP + asserts: + - matchRegex: + path: spec.template.spec.containers[0].command[2] + pattern: 'BIND_LISTENERS="CLIENT://\$\{POD_IP\}:9124, INTERNAL://\$\{POD_IP\}:9123"' + + - it: should follow custom listener ports + set: + listeners.internal.port: 19123 + listeners.client.port: 19124 + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: FLUSS_ADVERTISED_LISTENER_CLIENT + value: $(POD_NAME).coordinator-server-hs.$(POD_NAMESPACE).svc.cluster.local:19124 + - contains: + path: spec.template.spec.containers[0].env + content: + name: FLUSS_ADVERTISED_LISTENER_INTERNAL + value: $(POD_IP):19123 + + - it: should publish one env var per configured listener + set: + listeners.external.port: 9125 + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: FLUSS_ADVERTISED_LISTENER_EXTERNAL + value: $(POD_NAME).coordinator-server-hs.$(POD_NAMESPACE).svc.cluster.local:9125 + - matchRegex: + path: spec.template.spec.containers[0].command[2] + pattern: 'BIND_LISTENERS="CLIENT://\$\{POD_IP\}:9124, EXTERNAL://\$\{POD_IP\}:9125, INTERNAL://\$\{POD_IP\}:9123"' + - matchRegex: + path: spec.template.spec.containers[0].command[2] + pattern: 'EXTERNAL://\$\{FLUSS_ADVERTISED_LISTENER_EXTERNAL\}' + +--- + +suite: tablet advertised listener env +templates: + - templates/sts-tablet.yaml + +tests: + - it: should publish the CLIENT advertised address as an env var + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: FLUSS_ADVERTISED_LISTENER_CLIENT + value: $(POD_NAME).tablet-server-hs.$(POD_NAMESPACE).svc.cluster.local:9124 + + - it: should publish the INTERNAL advertised address as the pod IP + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: FLUSS_ADVERTISED_LISTENER_INTERNAL + value: $(POD_IP):9123 + + - it: should build advertised.listeners from the published env vars + asserts: + - matchRegex: + path: spec.template.spec.containers[0].command[2] + pattern: 'ADVERTISED_LISTENERS="CLIENT://\$\{FLUSS_ADVERTISED_LISTENER_CLIENT\}, INTERNAL://\$\{FLUSS_ADVERTISED_LISTENER_INTERNAL\}"' diff --git a/website/docs/install-deploy/deploying-with-helm.md b/website/docs/install-deploy/deploying-with-helm.md index 4a483e14a74..007b4614acd 100644 --- a/website/docs/install-deploy/deploying-with-helm.md +++ b/website/docs/install-deploy/deploying-with-helm.md @@ -587,6 +587,13 @@ security: mechanism: "" ``` +Each server pod publishes the advertised address of every listener as an environment variable named `FLUSS_ADVERTISED_LISTENER_`, with a `:` value. The server reads its `advertised.listeners` from these variables, so the published and the registered addresses are the same. With the default listeners, `tablet-server-0` in namespace `fluss` gets: + +``` +FLUSS_ADVERTISED_LISTENER_CLIENT=tablet-server-0.tablet-server-hs.fluss.svc.cluster.local:9124 +FLUSS_ADVERTISED_LISTENER_INTERNAL=:9123 +``` + ### Enabling Secure Connection With the helm deployment, you can specify authentication mechanisms when connecting to the Fluss cluster.