diff --git a/helm/templates/_helpers.tpl b/helm/templates/_helpers.tpl index 3933debca68..b58ec9beeae 100644 --- a/helm/templates/_helpers.tpl +++ b/helm/templates/_helpers.tpl @@ -131,3 +131,46 @@ imagePullSecrets: {{- end -}} {{- end -}} + +{{/* +Env entries publishing each listener's advertised address as +FLUSS_ADVERTISED_LISTENER_=:, one per entry in .Values.listeners. +Fluss advertises a listener missing from advertised.listeners at its bind address, +so INTERNAL is published as the pod IP; the others use the pod's headless-service DNS name. +The values reference POD_* via $(VAR), so they must be rendered after those entries. +Usage: include "fluss.listeners.advertisedEnv" (dict "root" . "service" "coordinator-server-hs") +*/}} +{{- define "fluss.listeners.advertisedEnv" -}} +{{- $service := .service -}} +{{- range $name, $listener := .root.Values.listeners }} +- name: FLUSS_ADVERTISED_LISTENER_{{ upper $name }} + {{- if eq $name "internal" }} + value: $(POD_IP):{{ $listener.port }} + {{- else }} + value: $(POD_NAME).{{ $service }}.$(POD_NAMESPACE).svc.cluster.local:{{ $listener.port }} + {{- end }} +{{- end }} +{{- end -}} + +{{/* +bind.listeners value: every listener bound on the pod IP. +*/}} +{{- define "fluss.listeners.bind" -}} +{{- $out := list -}} +{{- range $name, $listener := .Values.listeners -}} +{{- $out = append $out (printf "%s://${POD_IP}:%v" (upper $name) $listener.port) -}} +{{- end -}} +{{- join ", " $out -}} +{{- end -}} + +{{/* +advertised.listeners value, read from the env entries of "fluss.listeners.advertisedEnv" +so the published addresses and the configured ones cannot differ. +*/}} +{{- define "fluss.listeners.advertised" -}} +{{- $out := list -}} +{{- range $name, $_ := .Values.listeners -}} +{{- $out = append $out (printf "%s://${FLUSS_ADVERTISED_LISTENER_%s}" (upper $name) (upper $name)) -}} +{{- end -}} +{{- join ", " $out -}} +{{- end -}} diff --git a/helm/templates/sts-coordinator.yaml b/helm/templates/sts-coordinator.yaml index 583cfcd9cbd..ae530b80bc5 100644 --- a/helm/templates/sts-coordinator.yaml +++ b/helm/templates/sts-coordinator.yaml @@ -94,6 +94,7 @@ spec: valueFrom: fieldRef: fieldPath: metadata.namespace + {{- include "fluss.listeners.advertisedEnv" (dict "root" . "service" "coordinator-server-hs") | trim | nindent 12 }} - name: NODE_IP valueFrom: fieldRef: @@ -124,8 +125,8 @@ spec: export FLUSS_SERVER_ID=${POD_NAME##*-} && \ cp /opt/conf/server.yaml $FLUSS_HOME/conf && \ - BIND_LISTENERS="INTERNAL://${POD_IP}:{{ .Values.listeners.internal.port }}, CLIENT://${POD_IP}:{{ .Values.listeners.client.port }}" && \ - ADVERTISED_LISTENERS="CLIENT://${POD_NAME}.coordinator-server-hs.${POD_NAMESPACE}.svc.cluster.local:{{ .Values.listeners.client.port }}" && \ + BIND_LISTENERS="{{ include "fluss.listeners.bind" . }}" && \ + ADVERTISED_LISTENERS="{{ include "fluss.listeners.advertised" . }}" && \ echo "" >> $FLUSS_HOME/conf/server.yaml && \ echo "bind.listeners: ${BIND_LISTENERS}" >> $FLUSS_HOME/conf/server.yaml && \ diff --git a/helm/templates/sts-tablet.yaml b/helm/templates/sts-tablet.yaml index 5407b2a6a95..8776ffef322 100644 --- a/helm/templates/sts-tablet.yaml +++ b/helm/templates/sts-tablet.yaml @@ -94,6 +94,7 @@ spec: valueFrom: fieldRef: fieldPath: metadata.namespace + {{- include "fluss.listeners.advertisedEnv" (dict "root" . "service" "tablet-server-hs") | trim | nindent 12 }} {{- if (include "fluss.security.jaas.required" .) }} - name: FLUSS_ENV_JAVA_OPTS value: "-Djava.security.auth.login.config=/etc/fluss/conf/jaas.conf" @@ -120,8 +121,8 @@ spec: export FLUSS_SERVER_ID=${POD_NAME##*-} && \ cp /opt/conf/server.yaml $FLUSS_HOME/conf && \ - BIND_LISTENERS="INTERNAL://${POD_IP}:{{ .Values.listeners.internal.port }}, CLIENT://${POD_IP}:{{ .Values.listeners.client.port }}" && \ - ADVERTISED_LISTENERS="CLIENT://${POD_NAME}.tablet-server-hs.${POD_NAMESPACE}.svc.cluster.local:{{ .Values.listeners.client.port }}" && \ + BIND_LISTENERS="{{ include "fluss.listeners.bind" . }}" && \ + ADVERTISED_LISTENERS="{{ include "fluss.listeners.advertised" . }}" && \ echo "" >> $FLUSS_HOME/conf/server.yaml && \ echo "tablet-server.id: ${FLUSS_SERVER_ID}" >> $FLUSS_HOME/conf/server.yaml && \ diff --git a/helm/tests/listeners_env_test.yaml b/helm/tests/listeners_env_test.yaml new file mode 100644 index 00000000000..68d0d81b9f0 --- /dev/null +++ b/helm/tests/listeners_env_test.yaml @@ -0,0 +1,111 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +suite: coordinator advertised listener env +templates: + - templates/sts-coordinator.yaml + +tests: + - it: should publish the CLIENT advertised address as an env var + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: FLUSS_ADVERTISED_LISTENER_CLIENT + value: $(POD_NAME).coordinator-server-hs.$(POD_NAMESPACE).svc.cluster.local:9124 + + - it: should publish the INTERNAL advertised address as the pod IP + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: FLUSS_ADVERTISED_LISTENER_INTERNAL + value: $(POD_IP):9123 + + - it: should build advertised.listeners from the published env vars + asserts: + - matchRegex: + path: spec.template.spec.containers[0].command[2] + pattern: 'ADVERTISED_LISTENERS="CLIENT://\$\{FLUSS_ADVERTISED_LISTENER_CLIENT\}, INTERNAL://\$\{FLUSS_ADVERTISED_LISTENER_INTERNAL\}"' + + - it: should bind every listener on the pod IP + asserts: + - matchRegex: + path: spec.template.spec.containers[0].command[2] + pattern: 'BIND_LISTENERS="CLIENT://\$\{POD_IP\}:9124, INTERNAL://\$\{POD_IP\}:9123"' + + - it: should follow custom listener ports + set: + listeners.internal.port: 19123 + listeners.client.port: 19124 + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: FLUSS_ADVERTISED_LISTENER_CLIENT + value: $(POD_NAME).coordinator-server-hs.$(POD_NAMESPACE).svc.cluster.local:19124 + - contains: + path: spec.template.spec.containers[0].env + content: + name: FLUSS_ADVERTISED_LISTENER_INTERNAL + value: $(POD_IP):19123 + + - it: should publish one env var per configured listener + set: + listeners.external.port: 9125 + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: FLUSS_ADVERTISED_LISTENER_EXTERNAL + value: $(POD_NAME).coordinator-server-hs.$(POD_NAMESPACE).svc.cluster.local:9125 + - matchRegex: + path: spec.template.spec.containers[0].command[2] + pattern: 'BIND_LISTENERS="CLIENT://\$\{POD_IP\}:9124, EXTERNAL://\$\{POD_IP\}:9125, INTERNAL://\$\{POD_IP\}:9123"' + - matchRegex: + path: spec.template.spec.containers[0].command[2] + pattern: 'EXTERNAL://\$\{FLUSS_ADVERTISED_LISTENER_EXTERNAL\}' + +--- + +suite: tablet advertised listener env +templates: + - templates/sts-tablet.yaml + +tests: + - it: should publish the CLIENT advertised address as an env var + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: FLUSS_ADVERTISED_LISTENER_CLIENT + value: $(POD_NAME).tablet-server-hs.$(POD_NAMESPACE).svc.cluster.local:9124 + + - it: should publish the INTERNAL advertised address as the pod IP + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: FLUSS_ADVERTISED_LISTENER_INTERNAL + value: $(POD_IP):9123 + + - it: should build advertised.listeners from the published env vars + asserts: + - matchRegex: + path: spec.template.spec.containers[0].command[2] + pattern: 'ADVERTISED_LISTENERS="CLIENT://\$\{FLUSS_ADVERTISED_LISTENER_CLIENT\}, INTERNAL://\$\{FLUSS_ADVERTISED_LISTENER_INTERNAL\}"' diff --git a/website/docs/install-deploy/deploying-with-helm.md b/website/docs/install-deploy/deploying-with-helm.md index 4a483e14a74..007b4614acd 100644 --- a/website/docs/install-deploy/deploying-with-helm.md +++ b/website/docs/install-deploy/deploying-with-helm.md @@ -587,6 +587,13 @@ security: mechanism: "" ``` +Each server pod publishes the advertised address of every listener as an environment variable named `FLUSS_ADVERTISED_LISTENER_`, with a `:` value. The server reads its `advertised.listeners` from these variables, so the published and the registered addresses are the same. With the default listeners, `tablet-server-0` in namespace `fluss` gets: + +``` +FLUSS_ADVERTISED_LISTENER_CLIENT=tablet-server-0.tablet-server-hs.fluss.svc.cluster.local:9124 +FLUSS_ADVERTISED_LISTENER_INTERNAL=:9123 +``` + ### Enabling Secure Connection With the helm deployment, you can specify authentication mechanisms when connecting to the Fluss cluster.