Release v1.4.5: promote develop → main

.github/workflows/helm-ci.yaml

@@ -116,6 +116,30 @@ jobs:
             > /dev/null
           echo "Schema validation passed for ${{ matrix.platform }}"
 
+  ingestor-multiarch:
+    # Guard: the chart's PINNED ingestor digest must be a multi-arch index
+    # (linux/amd64 + linux/arm64). Greenfield installs spawn the ingestor from this
+    # pinned digest (before image-refresh ticks), so an amd64-only pin breaks data
+    # ingestion on arm64 hosts (Apple Silicon, Graviton) with ImagePullBackOff. This
+    # would have caught #160 (which pinned the amd64-only v0.3.1). See client#186.
+    name: Pinned ingestor digest is multi-arch
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Assert images.ingestor.digest supports linux/amd64 + linux/arm64
+        run: |
+          digest=$(yq '.images.ingestor.digest' client/values.yaml)
+          echo "Pinned ingestor digest: $digest"
+          if [ -z "$digest" ] || [ "$digest" = "null" ]; then
+            echo "::error::images.ingestor.digest is empty — it must be a pinned multi-arch digest."; exit 1
+          fi
+          plats=$(docker buildx imagetools inspect "ghcr.io/tracebloc/ingestor@$digest" 2>&1 \
+            | awk '/Platform:/{print $2}' | grep -v '^unknown' | sort -u)
+          echo "Platforms: $(echo "$plats" | paste -sd' ' -)"
+          echo "$plats" | grep -qx 'linux/arm64' || { echo "::error::Pinned ingestor digest is NOT multi-arch (no linux/arm64). arm64 installs (Apple Silicon, Graviton) would fail ingestion with 'no match for platform' / ImagePullBackOff. Pin a multi-arch :0.x index. See client#186 / #160."; exit 1; }
+          echo "$plats" | grep -qx 'linux/amd64' || { echo "::error::Pinned ingestor digest is missing linux/amd64."; exit 1; }
+          echo "OK — pinned ingestor digest is multi-arch (amd64 + arm64)."
+
   # Installer script tests (bats + Pester) + the cross-distro prerequisite matrix
   # live in their own workflow: .github/workflows/installer-tests.yaml
   # (triggered on scripts/** changes).

client/Chart.yaml

@@ -2,8 +2,8 @@ apiVersion: v2
 name: client
 description: A unified Helm chart for tracebloc on AKS, EKS, bare-metal, and OpenShift
 type: application
-version: 1.4.4
-appVersion: "1.4.4"
+version: 1.4.5
+appVersion: "1.4.5"
 keywords:
   - tracebloc
   - kubernetes

client/templates/image-refresh-cronjob.yaml

@@ -83,6 +83,12 @@ data:
     log "  client images: tracebloc/{jobs-manager,pods-monitor} on docker.io under tag=$IMAGE_TAG"
     log "  ingestor image: tracebloc/ingestor on ghcr.io under tag=$INGESTOR_TAG (pinned=$INGESTOR_PINNED)"
 
+    # Consecutive failed ghcr ingestor-digest resolves before image-refresh
+    # escalates from WARN+skip to a failed Job (#186 #2). The chart injects
+    # this from imageRefresh.ingestorResolveFailureThreshold; default here
+    # too so a hand-edited CronJob missing the env var still runs under -u.
+    : "${INGESTOR_RESOLVE_FAIL_THRESHOLD:=3}"
+
     # Get an anonymous pull-scope token for one repository on a
     # registry. Both docker.io and ghcr.io support anonymous tokens for
     # public images; only the issuer URL differs. tr cleanup handles
@@ -284,9 +290,50 @@ data:
       log "  ingestor.autoRefresh: false in values; skipping (operator opted into pinning)"
     else
       latest_ingestor="$(get_latest_digest "tracebloc/ingestor" "$INGESTOR_TAG" "ghcr.io" || true)"
+      ingestor_fail_key="tracebloc.io/ingestor-refresh-consecutive-failures"
+      ingestor_err_key="tracebloc.io/ingestor-refresh-last-error"
       if [ -z "$latest_ingestor" ]; then
-        log "  WARN: could not resolve latest digest (rate-limited or transient); skipping this tick"
+        # Could not resolve the ingestor digest from ghcr.io. A transient
+        # blip (rate-limit, momentary DNS) and a PERSISTENT failure (egress
+        # to ghcr.io firewalled, the ghcr token endpoint blocked, a proxy
+        # that allowlists docker.io but not ghcr.io) are indistinguishable
+        # on a single tick — so COUNT consecutive failures and escalate
+        # once they cross the threshold, instead of skipping silently
+        # forever. #186 (#2): berlin-team sat on the amd64-only baseline
+        # because every ingestor tick hit this branch while the docker.io
+        # images (jobs-manager, pods-monitor) refreshed fine — the CronJob
+        # looked healthy and nothing surfaced the ghcr failure.
+        prev_fails="$(get_annotation "$ingestor_fail_key" || true)"
+        case "$prev_fails" in ''|*[!0-9]*) prev_fails=0 ;; esac
+        if [ "$prev_fails" -ge "$INGESTOR_RESOLVE_FAIL_THRESHOLD" ]; then
+          # Already escalated on an earlier tick; stay loud (fail the Job)
+          # without inflating the counter — it caps at the threshold.
+          log "  ERROR: still cannot resolve ghcr.io/tracebloc/ingestor:${INGESTOR_TAG} (>= ${INGESTOR_RESOLVE_FAIL_THRESHOLD} consecutive failures). Ingestor digest is NOT auto-refreshing; ingestion may be stuck on a stale image. Check egress to ghcr.io and its token endpoint from this namespace. See client#186."
+          exit 1
+        fi
+        fails=$((prev_fails + 1))
+        kubectl annotate deployment -n "$RELEASE_NAMESPACE" "$DEPLOYMENT_NAME" \
+          "${ingestor_fail_key}=${fails}" --overwrite
+        if [ "$fails" -ge "$INGESTOR_RESOLVE_FAIL_THRESHOLD" ]; then
+          # Persistent. Record a human-readable last-error and fail the Job
+          # so it surfaces in `kubectl get cronjob` and monitoring — the
+          # same "failed Job = operator-visible" idiom Pass 2's stuck-
+          # rollout check already relies on.
+          kubectl annotate deployment -n "$RELEASE_NAMESPACE" "$DEPLOYMENT_NAME" \
+            "${ingestor_err_key}=could not resolve ghcr.io/tracebloc/ingestor:${INGESTOR_TAG} for ${fails} consecutive ticks" --overwrite
+          log "  ERROR: could not resolve ghcr.io/tracebloc/ingestor:${INGESTOR_TAG} for ${fails} consecutive ticks (threshold ${INGESTOR_RESOLVE_FAIL_THRESHOLD}). Ingestor digest is NOT auto-refreshing; ingestion may be stuck on a stale image (e.g. an amd64-only baseline on arm64 nodes). Check egress to ghcr.io and its token endpoint from this namespace. Failing the Job so this surfaces in monitoring. See client#186."
+          exit 1
+        fi
+        log "  WARN: could not resolve latest ingestor digest (failure ${fails}/${INGESTOR_RESOLVE_FAIL_THRESHOLD}; transient?); skipping this tick"
       else
+        # Resolved OK. Clear any prior failure streak so a recovered
+        # registry/egress blip doesn't leave a stale failure annotation
+        # (and a future failure starts counting from zero again).
+        if [ -n "$(get_annotation "$ingestor_fail_key" || true)" ]; then
+          log "  ingestor digest resolved; clearing prior failure streak"
+          kubectl annotate deployment -n "$RELEASE_NAMESPACE" "$DEPLOYMENT_NAME" \
+            "${ingestor_fail_key}-" "${ingestor_err_key}-" >/dev/null 2>&1 || true
+        fi
         recorded_ingestor="$(get_annotation "tracebloc.io/last-refreshed-ingestor-digest" || true)"
         # Always read spec env too — the annotation alone isn't enough
         # because external actors can revert the spec without touching
@@ -484,6 +531,13 @@ spec:
                 # semver-style float tags. Caught in PR #162 review.
                 - name: INGESTOR_TAG
                   value: {{ (default dict (default dict .Values.images).ingestor).tag | default "0.3" | quote }}
+                # Consecutive failed ghcr ingestor-digest resolves before
+                # image-refresh stops silently skipping and fails the Job
+                # loudly (#186 #2). nil-guarded with a `default 3` fallback
+                # so --reuse-values upgrades from pre-this-PR stored
+                # manifests (which lack the key) still render.
+                - name: INGESTOR_RESOLVE_FAIL_THRESHOLD
+                  value: {{ (default dict .Values.imageRefresh).ingestorResolveFailureThreshold | default 3 | quote }}
               securityContext:
                 allowPrivilegeEscalation: false
                 readOnlyRootFilesystem: true

client/templates/network-policy-training.yaml

@@ -85,4 +85,19 @@ spec:
       ports:
         - port: 3306
           protocol: TCP
+    # 4. requests-proxy — training pods POST epoch results / FLOPs to the
+    #    in-namespace requests-proxy on 8888 (so they never hold Service Bus
+    #    credentials). Rule 2 blocks this ClusterIP egress, so re-permit it
+    #    explicitly, exactly like MySQL above. Without this rule every
+    #    experiment CrashLoopBackOffs at the first epoch finalize with
+    #    "requests-proxy-service:8888 ... Connection refused" (client#196).
+    #    Service selector + port: templates/requests-proxy-service.yaml
+    #    (app=requests-proxy, 8888).
+    - to:
+        - podSelector:
+            matchLabels:
+              app: requests-proxy
+      ports:
+        - port: 8888
+          protocol: TCP
 {{- end }}

client/templates/rbac.yaml

@@ -44,7 +44,12 @@ rules:
     verbs: ["create"]
   - apiGroups: [""]
     resources: ["configmaps", "secrets"]
-    verbs: ["create"]
+    # `get` is required by jobs-manager's orphan-resource verify path
+    # (client-runtime#52) and the missing-row self-heal (client-runtime#54):
+    # on a create-409 it reads the existing ConfigMap/Secret to confirm the
+    # content matches before reusing it. Without `get`, those reads return
+    # Forbidden and the endpoint 500s instead of the intended 409/replay.
+    verbs: ["create", "get"]
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRoleBinding
@@ -95,7 +100,12 @@ rules:
   # to authenticate ingestion calls — see the ClusterRole branch above.
   - apiGroups: [""]
     resources: ["configmaps", "secrets"]
-    verbs: ["create"]
+    # `get` is required by jobs-manager's orphan-resource verify path
+    # (client-runtime#52) and the missing-row self-heal (client-runtime#54):
+    # on a create-409 it reads the existing ConfigMap/Secret to confirm the
+    # content matches before reusing it. Without `get`, those reads return
+    # Forbidden and the endpoint 500s instead of the intended 409/replay.
+    verbs: ["create", "get"]
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: RoleBinding

client/templates/resource-monitor-rbac.yaml

@@ -11,7 +11,24 @@ metadata:
   labels:
     {{- include "tracebloc.labels" . | nindent 4 }}
     app: {{ include "tracebloc.resourceMonitorName" . }}
-{{- if ne .Values.clusterScope false }}
+{{/*
+  Per-node monitoring is INTRINSICALLY cluster-scoped, so the resource-monitor
+  always needs a ClusterRole -- it is deliberately NOT gated on .Values.clusterScope.
+  The code path that requires it:
+    * resource_monitor.py uses core_v1_api.list_pod_for_all_namespaces(
+        field_selector="spec.nodeName=<node>") to enumerate every pod on the node.
+      list_pod_for_all_namespaces is a CLUSTER-SCOPED list verb that a namespaced
+      Role can never satisfy (it 403s -> CrashLoopBackOff).
+    * It also read_namespaced_pod()s its OWN pod, which lives in
+      .Values.nodeAgents.namespace.name (NOT .Release.Namespace), so a Role scoped
+      to the release namespace would miss it too.
+  This ClusterRole is strictly READ-ONLY (get/list/watch on pod/node metadata +
+  metrics); it grants no write/exec/secret access. clusterScope continues to gate
+  the training/jobs isolation footprint elsewhere -- it must not cripple node
+  telemetry by leaving the DaemonSet without the permissions it cannot run without.
+  If a deployment genuinely cannot allow any cluster-scoped read, disable the
+  monitor entirely via .Values.resourceMonitor=false rather than deploying it broken.
+*/}}
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRole
@@ -49,49 +66,4 @@ subjects:
   - kind: ServiceAccount
     name: {{ include "tracebloc.resourceMonitorName" . }}
     namespace: {{ .Values.nodeAgents.namespace.name }}
-{{- else }}
----
-# Role + RoleBinding live in the RELEASE namespace so the resource-monitor
-# can list pods/logs where the actual workloads run. The RoleBinding
-# subject points at the ServiceAccount in the node-agents namespace
-# (cross-namespace bindings are valid; the Role scope is what matters).
-apiVersion: rbac.authorization.k8s.io/v1
-kind: Role
-metadata:
-  name: tracebloc-resource-monitor-{{ .Release.Name }}
-  namespace: {{ .Release.Namespace }}
-  annotations:
-    meta.helm.sh/release-name: {{ .Release.Name }}
-    meta.helm.sh/release-namespace: {{ .Release.Namespace }}
-  labels:
-    {{- include "tracebloc.labels" . | nindent 4 }}
-    app: {{ include "tracebloc.resourceMonitorName" . }}
-rules:
-  - apiGroups: [""]
-    resources: ["pods", "pods/log"]
-    verbs: ["get", "list", "watch"]
-  - apiGroups: ["metrics.k8s.io"]
-    resources: ["pods"]
-    verbs: ["get", "list", "watch"]
----
-apiVersion: rbac.authorization.k8s.io/v1
-kind: RoleBinding
-metadata:
-  name: tracebloc-resource-monitor-{{ .Release.Name }}
-  namespace: {{ .Release.Namespace }}
-  annotations:
-    meta.helm.sh/release-name: {{ .Release.Name }}
-    meta.helm.sh/release-namespace: {{ .Release.Namespace }}
-  labels:
-    {{- include "tracebloc.labels" . | nindent 4 }}
-    app: {{ include "tracebloc.resourceMonitorName" . }}
-roleRef:
-  apiGroup: rbac.authorization.k8s.io
-  kind: Role
-  name: tracebloc-resource-monitor-{{ .Release.Name }}
-subjects:
-  - kind: ServiceAccount
-    name: {{ include "tracebloc.resourceMonitorName" . }}
-    namespace: {{ .Values.nodeAgents.namespace.name }}
-{{- end }}
 {{- end }}

client/tests/image_refresh_test.yaml

@@ -463,6 +463,56 @@ tests:
             name: INGESTOR_TAG
             value: staging
 
+  - it: ingestor INGESTOR_RESOLVE_FAIL_THRESHOLD defaults to "3" (#186 #2)
+    # Below this many consecutive ghcr resolve failures, image-refresh
+    # WARNs + skips; at/above it the script fails the Job loudly instead
+    # of silently leaving jobs-manager on a stale digest.
+    template: templates/image-refresh-cronjob.yaml
+    documentIndex: 1
+    asserts:
+      - contains:
+          path: spec.jobTemplate.spec.template.spec.containers[0].env
+          content:
+            name: INGESTOR_RESOLVE_FAIL_THRESHOLD
+            value: "3"
+
+  - it: ingestor INGESTOR_RESOLVE_FAIL_THRESHOLD falls back to "3" when imageRefresh key is absent
+    # --reuse-values upgrade from a pre-this-PR stored manifest lacks the
+    # key; the template's `| default 3` must still render a usable value.
+    template: templates/image-refresh-cronjob.yaml
+    documentIndex: 1
+    set:
+      imageRefresh:
+        ingestorResolveFailureThreshold: null
+    asserts:
+      - contains:
+          path: spec.jobTemplate.spec.template.spec.containers[0].env
+          content:
+            name: INGESTOR_RESOLVE_FAIL_THRESHOLD
+            value: "3"
+
+  - it: ingestor INGESTOR_RESOLVE_FAIL_THRESHOLD is overridable
+    template: templates/image-refresh-cronjob.yaml
+    documentIndex: 1
+    set:
+      imageRefresh:
+        ingestorResolveFailureThreshold: 5
+    asserts:
+      - contains:
+          path: spec.jobTemplate.spec.template.spec.containers[0].env
+          content:
+            name: INGESTOR_RESOLVE_FAIL_THRESHOLD
+            value: "5"
+
+  - it: schema rejects ingestorResolveFailureThreshold below 1
+    template: templates/image-refresh-cronjob.yaml
+    set:
+      imageRefresh:
+        ingestorResolveFailureThreshold: 0
+    asserts:
+      - failedTemplate:
+          errorPattern: "must not be valid against schema|Must be greater than or equal to 1"
+
   - it: schema rejects ingestor.tag=latest
     template: templates/image-refresh-cronjob.yaml
     set:

client/tests/mysql_test.yaml

@@ -61,8 +61,18 @@ tests:
           path: spec.template.spec.containers[0].livenessProbe.failureThreshold
           value: 5
 
-  - it: should reference the data-plane PriorityClass when enabled
+  - it: should NOT set priorityClassName by default (PriorityClass dropped)
     template: templates/mysql-deployment.yaml
+    asserts:
+      - notExists:
+          path: spec.template.spec.priorityClassName
+
+  - it: should reference the PriorityClass when a name is set
+    template: templates/mysql-deployment.yaml
+    set:
+      priorityClass:
+        create: true
+        name: tracebloc-data-plane
     asserts:
       - equal:
           path: spec.template.spec.priorityClassName

client/tests/node_agents_namespace_test.yaml

@@ -154,28 +154,30 @@ tests:
           path: subjects[0].namespace
           value: tracebloc-node-agents
 
-  - it: should keep namespace-scoped Role + RoleBinding in the release namespace when clusterScope is false, with SA subject in node-agents
+  - it: should still render a cluster-scoped ClusterRole + ClusterRoleBinding when clusterScope is false, with SA subject in node-agents
     template: templates/resource-monitor-rbac.yaml
     set:
       clusterScope: false
     asserts:
-      # Role must grant access where the monitored workloads live (release ns)
+      # Per-node monitoring relies on list_pod_for_all_namespaces (a cluster-scoped
+      # verb a namespaced Role can never satisfy) and reads its own pod in the
+      # node-agents namespace, so resource-monitor RBAC is intentionally decoupled
+      # from clusterScope -- it is ALWAYS cluster-scoped. clusterScope still gates
+      # the training/jobs isolation footprint elsewhere.
       - isKind:
-          of: Role
+          of: ClusterRole
         documentIndex: 1
-      - equal:
+      # Cluster-scoped resources carry no metadata.namespace
+      - isNull:
           path: metadata.namespace
-          value: tracebloc-templates
         documentIndex: 1
-      # RoleBinding sits in the release ns so it applies the Role there
       - isKind:
-          of: RoleBinding
+          of: ClusterRoleBinding
         documentIndex: 2
-      - equal:
+      - isNull:
           path: metadata.namespace
-          value: tracebloc-templates
         documentIndex: 2
-      # ...but the subject SA lives in the node-agents namespace
+      # ...but the subject SA still lives in the node-agents namespace
       - equal:
           path: subjects[0].namespace
           value: tracebloc-node-agents

client/tests/priority_class_pdb_test.yaml

@@ -7,8 +7,18 @@ set:
   clientId: "test-id"
   clientPassword: "test"
 tests:
-  - it: should create the data-plane PriorityClass by default
+  - it: should NOT render the PriorityClass by default (dropped)
     template: templates/priority-class.yaml
+    asserts:
+      - hasDocuments:
+          count: 0
+
+  - it: should render the PriorityClass when explicitly enabled
+    template: templates/priority-class.yaml
+    set:
+      priorityClass:
+        create: true
+        name: tracebloc-data-plane
     asserts:
       - isKind:
           of: PriorityClass