feat(pangolin): transitional Gerbil host gateway mode and cross-namespace Badger fix

.github/e2e/kind-cluster.yaml

@@ -0,0 +1,8 @@
+# Three nodes so "Traefik landed on the Gerbil node" is a real assertion rather than the
+# only possible outcome.
+kind: Cluster
+apiVersion: kind.x-k8s.io/v1alpha4
+nodes:
+  - role: control-plane
+  - role: worker
+  - role: worker

.github/e2e/values-host-gateway.yaml

@@ -0,0 +1,61 @@
+# Values for the host gateway data-path check in .github/workflows/pangolin-tunnel-e2e.yaml.
+#
+# Standalone + multi so the chart renders its own Traefik and the co-location affinity is
+# actually exercised. SQLite keeps the install to a single workload with no database
+# dependency; this run is about the network path, not persistence.
+deployment:
+  type: standalone
+  mode: multi
+
+controller:
+  enabled: false
+
+database:
+  mode: sqlite
+
+networkPolicy:
+  # kind's default CNI does not enforce NetworkPolicy, so rendering it here would prove
+  # nothing. Its rules are covered by the unit tests instead.
+  enabled: false
+
+namespace:
+  create: false
+
+pangolin:
+  config:
+    app:
+      dashboard_url: https://pangolin.example.test
+    domains:
+      domain1:
+        base_domain: example.test
+        cert_resolver: letsencrypt
+    gerbil:
+      base_endpoint: gerbil.example.test
+
+gerbil:
+  enabled: true
+  startupMode: normal
+  replicaCount: 1
+  hostGateway:
+    enabled: true
+  waitForPangolin:
+    enabled: false
+  persistence:
+    enabled: false
+  service:
+    type: ClusterIP
+
+traefik:
+  enabled: true
+  colocateWithGerbil: required
+  persistence:
+    enabled: false
+  config:
+    letsencryptEmail: ops@example.test
+    dynamicRouters:
+      host: pangolin.example.test
+
+cnpg-operator:
+  enabled: false
+cnpg-cluster:
+  enabled: false

.github/workflows/helm-ci.yaml

@@ -177,6 +177,49 @@ jobs:
           print("pangolin ingress ports:", sorted(in_ports))
           PY
 
+      - name: Verify every default image tag exists in its registry
+        run: |
+          set -euo pipefail
+          # A chart whose defaults point at an unpublished image must not merge.
+          #
+          # TODO: remove ALLOW_PENDING once
+          # ghcr.io/fosrl/pangolin-kube-controller:0.1.0-alpha.2 is published.
+          # Exactly one image is tolerated, and only on dev (pull_request into dev, or
+          # push to dev). main and every other image stay fatal.
+          ALLOW_PENDING="ghcr.io/fosrl/pangolin-kube-controller:0.1.0-alpha.2"
+          allow_pending=false
+          if [ "${GITHUB_BASE_REF:-}" = "dev" ] || [ "${GITHUB_REF_NAME:-}" = "dev" ]; then
+            allow_pending=true
+          fi
+          missing=0
+          while read -r image; do
+            [ -n "${image}" ] || continue
+            repo="${image%:*}"; tag="${image##*:}"
+            case "${repo}" in
+              ghcr.io/*)
+                registry="ghcr.io"; path="${repo#ghcr.io/}"
+                token_url="https://ghcr.io/token?scope=repository:${path}:pull&service=ghcr.io" ;;
+              docker.io/*)
+                registry="registry-1.docker.io"; path="${repo#docker.io/}"
+                token_url="https://auth.docker.io/token?service=registry.docker.io&scope=repository:${path}:pull" ;;
+              *) echo "skip (unknown registry): ${image}"; continue ;;
+            esac
+            token=$(curl -fsSL "${token_url}" | python3 -c 'import sys,json;print(json.load(sys.stdin).get("token",""))')
+            code=$(curl -s -o /dev/null -w '%{http_code}'               -H "Authorization: Bearer ${token}"               -H 'Accept: application/vnd.oci.image.index.v1+json,application/vnd.docker.distribution.manifest.list.v2+json,application/vnd.docker.distribution.manifest.v2+json'               "https://${registry}/v2/${path}/manifests/${tag}")
+            if [ "${code}" = "200" ]; then
+              echo "ok      ${image}"
+            elif [ "${image}" = "${ALLOW_PENDING}" ] && [ "${allow_pending}" = "true" ]; then
+              echo "KNOWN-PENDING ${image} (HTTP ${code})"
+            else
+              echo "MISSING ${image} (HTTP ${code})"
+              missing=$((missing + 1))
+            fi
+          done < <(helm template demo charts/pangolin --namespace pangolin --set database.mode=sqlite --set deployment.type=controller                     | grep -oE 'image: "?[a-z0-9./-]+:[a-zA-Z0-9._-]+"?'                     | sed -e 's/^image: //' -e 's/"//g' | sort -u)
+          if [ "${missing}" -gt 0 ]; then
+            echo "ERROR: ${missing} default image(s) are not published; this chart must not be released."
+            exit 1
+          fi
+
       - name: Verify image tags match appVersion
         run: |
           set -euo pipefail

.github/workflows/pangolin-tunnel-e2e.yaml

@@ -0,0 +1,203 @@
+name: Pangolin tunnel data path
+
+# What this covers, and what it does not.
+#
+# The tunnel-gate job is deterministic and always runs: it proves the guard in
+# scripts/e2e/assert-tunnel-backends.sh actually detects an unroutable tunnel address,
+# and that it refuses to report success when it inspected nothing or could not parse its
+# input - the two ways a check like this passes without checking anything.
+#
+# The host-gateway job needs a cluster and is therefore label-gated. It verifies the parts
+# of the data path a cluster can confirm without a live Pangolin/Newt pair: that Gerbil
+# really lands in the node network namespace, that the WireGuard route exists there, and
+# that Traefik is scheduled onto the same node.
+#
+# It does NOT verify a request traversing the tunnel. That needs a real Newt credential
+# and a registered site, so the 502 -> 200 assertion behind fosrl/helm-charts#20 is not
+# covered here. Do not read a green run as proof the tunnel works.
+
+on:
+  pull_request:
+  push:
+    branches: [main]
+
+permissions:
+  contents: read
+
+jobs:
+  tunnel-gate:
+    name: Tunnel backend guard self-test
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+
+      - name: An unroutable tunnel address must fail the guard
+        run: |
+          set -euo pipefail
+          if bash scripts/e2e/assert-tunnel-backends.sh \
+               --file scripts/e2e/testdata/endpointslices-tunnel.json \
+               --tunnel-cidr 100.89.128.0/20; then
+            echo "ERROR: the guard passed on an address inside the tunnel subnet"
+            exit 1
+          fi
+          echo "Guard correctly rejected the unroutable address."
+
+      - name: Routable addresses must pass
+        run: |
+          set -euo pipefail
+          bash scripts/e2e/assert-tunnel-backends.sh \
+            --file scripts/e2e/testdata/endpointslices-routable.json \
+            --tunnel-cidr 100.89.128.0/20
+
+      - name: An empty endpoint set must fail, because it checked nothing
+        run: |
+          set -euo pipefail
+          if bash scripts/e2e/assert-tunnel-backends.sh \
+               --file scripts/e2e/testdata/endpointslices-empty.json \
+               --tunnel-cidr 100.89.128.0/20; then
+            echo "ERROR: the guard reported success without inspecting a single address"
+            exit 1
+          fi
+          echo "Guard correctly refused to pass on an empty endpoint set."
+
+      - name: An empty endpoint set passes only when explicitly expected
+        run: |
+          set -euo pipefail
+          bash scripts/e2e/assert-tunnel-backends.sh \
+            --file scripts/e2e/testdata/endpointslices-empty.json \
+            --tunnel-cidr 100.89.128.0/20 \
+            --min-endpoints 0
+
+      - name: Unparseable input must fail, not read as empty
+        run: |
+          set -euo pipefail
+          printf %s "not json" > /tmp/broken.json
+          if bash scripts/e2e/assert-tunnel-backends.sh \
+               --file /tmp/broken.json \
+               --tunnel-cidr 100.89.128.0/20; then
+            echo "ERROR: the guard passed on input it could not parse"
+            exit 1
+          fi
+          echo "Guard correctly failed on unparseable input."
+
+  host-gateway:
+    name: Host gateway data path (kind)
+    # Cluster-backed and slow, so it runs on main and on demand via the label.
+    if: >-
+      github.event_name == 'push' ||
+      contains(github.event.pull_request.labels.*.name, 'e2e')
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+
+      - name: Set up Helm
+        uses: Azure/setup-helm@dda3372f752e03dde6b3237bc9431cdc2f7a02a2 # v5.0.0
+        with:
+          version: v3.20.2
+
+      - name: Create a multi-node kind cluster
+        uses: helm/kind-action@a1b0e391336a6ee6713a0583f8c6240d70863de3 # v1.12.0
+        with:
+          cluster_name: pangolin-e2e
+          # More than one node, so "Traefik landed on the Gerbil node" is a real
+          # assertion rather than the only possible outcome.
+          config: .github/e2e/kind-cluster.yaml
+
+      - name: Add Helm dependency repositories
+        run: |
+          helm repo add cloudnative-pg https://cloudnative-pg.github.io/charts
+          helm repo update
+
+      - name: Build chart dependencies
+        run: helm dependency build charts/pangolin
+
+      - name: Install the chart in host gateway mode
+        run: |
+          set -euo pipefail
+          kubectl create namespace pangolin-e2e
+          kubectl label namespace pangolin-e2e pod-security.kubernetes.io/enforce=privileged --overwrite
+          helm install pangolin charts/pangolin \
+            --namespace pangolin-e2e \
+            -f .github/e2e/values-host-gateway.yaml \
+            --wait --timeout 10m
+
+      - name: Gerbil must run in the node network namespace
+        run: |
+          set -euo pipefail
+          kubectl -n pangolin-e2e rollout status deploy/pangolin-gerbil --timeout=5m
+          gerbil_pods=$(kubectl -n pangolin-e2e get pods -l app.kubernetes.io/component=gerbil \
+            -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}')
+          if [ -z "${gerbil_pods}" ]; then
+            echo "ERROR: no Gerbil Pods matched the selector - this check would prove nothing"
+            exit 1
+          fi
+          host_network=$(kubectl -n pangolin-e2e get pods -l app.kubernetes.io/component=gerbil \
+            -o jsonpath='{.items[0].spec.hostNetwork}')
+          if [ "$host_network" != "true" ]; then
+            echo "ERROR: Gerbil is not using hostNetwork (got '${host_network}')"
+            exit 1
+          fi
+          dns_policy=$(kubectl -n pangolin-e2e get pods -l app.kubernetes.io/component=gerbil \
+            -o jsonpath='{.items[0].spec.dnsPolicy}')
+          if [ "$dns_policy" != "ClusterFirstWithHostNet" ]; then
+            echo "ERROR: dnsPolicy is '${dns_policy}'; Gerbil could not resolve the Pangolin Service"
+            exit 1
+          fi
+          echo "Gerbil: hostNetwork=true dnsPolicy=${dns_policy}"
+
+      - name: The WireGuard interface must exist on the node, not in a Pod namespace
+        run: |
+          set -euo pipefail
+          node=$(kubectl -n pangolin-e2e get pods -l app.kubernetes.io/component=gerbil \
+            -o jsonpath='{.items[0].spec.nodeName}')
+          echo "Gerbil node: ${node}"
+          # kind nodes are containers, so the node network namespace is inspectable directly.
+          if ! docker exec "${node}" ip link show wg0; then
+            echo "ERROR: wg0 is absent from the node network namespace - the whole point of this mode"
+            exit 1
+          fi
+          docker exec "${node}" ip route show | tee /tmp/node-routes.txt
+          if ! grep -q "wg0" /tmp/node-routes.txt; then
+            echo "ERROR: no route via wg0 in the node routing table"
+            exit 1
+          fi
+
+      - name: Traefik must be co-located with Gerbil
+        run: |
+          set -euo pipefail
+          gerbil_node=$(kubectl -n pangolin-e2e get pods -l app.kubernetes.io/component=gerbil \
+            -o jsonpath='{.items[0].spec.nodeName}')
+          # The chart labels its own Traefik `traefik-standalone`; a wrong selector here
+          # would make this check pass without comparing anything.
+          traefik_nodes=$(kubectl -n pangolin-e2e get pods -l app.kubernetes.io/component=traefik-standalone \
+            -o jsonpath='{range .items[*]}{.spec.nodeName}{"\n"}{end}' | sort -u)
+          echo "Gerbil: ${gerbil_node}"
+          echo "Traefik: ${traefik_nodes}"
+          if [ -z "${traefik_nodes}" ]; then
+            echo "ERROR: no Traefik Pods matched - the selector is wrong and this check proved nothing"
+            exit 1
+          fi
+          for node in ${traefik_nodes}; do
+            if [ "${node}" != "${gerbil_node}" ]; then
+              echo "ERROR: Traefik on ${node} cannot reach tunnel backends held by ${gerbil_node}"
+              exit 1
+            fi
+          done
+
+      - name: No unroutable tunnel address may be published
+        run: |
+          set -euo pipefail
+          bash scripts/e2e/assert-tunnel-backends.sh \
+            --namespace pangolin-e2e \
+            --tunnel-cidr 100.89.128.0/20 \
+            --expect-data-path
+
+      - name: Collect diagnostics
+        if: failure()
+        run: |
+          kubectl -n pangolin-e2e get pods -o wide || true
+          kubectl -n pangolin-e2e get endpointslices -o yaml || true
+          kubectl -n pangolin-e2e describe pods || true
+          kubectl -n pangolin-e2e logs -l app.kubernetes.io/component=gerbil --tail=200 || true

CHANGELOG.md

@@ -13,6 +13,12 @@ This changelog is chart-scoped to support multiple charts over time.
 
 #### Fixed
 
+- `pangolin.config.server.internal_hostname` now defaults to the in-cluster FQDN instead
+  of the bare Service name. Badger resolves it from inside the Traefik Pod, so with
+  `deployment.traefikNamespace` set it did not resolve at all and Badger answered
+  `404 page not found` for every request while the Traefik dashboard still reported the
+  router healthy. The same value drives the Pangolin UI and AI gateway URLs
+  ([#20](https://github.com/fosrl/helm-charts/issues/20)).
 - `pangolin.extraVolumes` / `pangolin.extraVolumeMounts` no longer abort `helm template`.
   The `- {{- toYaml . | nindent N }}` pattern let the whitespace strip eat the list dash
   ([#25](https://github.com/fosrl/helm-charts/issues/25)).
@@ -56,11 +62,19 @@ This changelog is chart-scoped to support multiple charts over time.
   loopback there. In controller mode with an external Traefik, NOTES says at install time
   that the port has no source and what to set, instead of letting AI gateway routes time
   out silently.
+- `gerbil.hostGateway.enabled`, a transitional opt-in mode that runs Gerbil with
+  `hostNetwork` so tunnel routes land in the node network namespace, plus
+  `traefik.colocateWithGerbil` to pin the chart-managed Traefik to that node and
+  `networkPolicy.gerbil.hostGateway.nodeCIDRs` for the node-sourced NetworkPolicy rule.
+  `PANGOLIN-071`..`075` reject the combinations that cannot work. This is a workaround,
+  not the portable fix: [#20](https://github.com/fosrl/helm-charts/issues/20) stays open.
+- `global.clusterDomain` for clusters that do not use `cluster.local`.
 
 #### Changed
 
-- Bumped Pangolin appVersion to `1.22.2`, the Gerbil image to `1.5.1` and the chart to
-  `0.1.0-alpha.2`. The chart remains a prerelease.
+- Bumped Pangolin appVersion to `1.22.2`, the Gerbil image to `1.5.1`, the
+  pangolin-kube-controller image to `0.1.0-alpha.2` and the chart to `0.1.0-alpha.2`.
+  The chart remains a prerelease.
 - `images.traefik.tag` moves from `v3.6.15` to `v3.7.13`. The v3.6 line is out of upstream
   security support, and the bundled Traefik chart already installs v3.7.13, so the two ways
   this chart can run Traefik no longer sit on different minors.

VERSION_MATRIX.md

@@ -25,7 +25,7 @@ Sources:
 | Kubernetes version | `>=1.30.14-0` |
 | Pangolin default image tag | `1.22.2` |
 | Pangolin PostgreSQL image tag | `postgresql-1.22.2` |
-| pangolin-kube-controller tag | `0.1.0-alpha.1` |
+| pangolin-kube-controller tag | `0.1.0-alpha.2` |
 | Gerbil tag | `1.5.1` |
 | Traefik tag | `v3.7.13` |