From 629773d21bed168ea054c1e19b6d8bafc81832a8 Mon Sep 17 00:00:00 2001 From: shujaat hasan Date: Tue, 2 Jun 2026 14:12:59 +0200 Subject: [PATCH 1/9] fix(resource-monitor): always grant read-only ClusterRole (decouple from clusterScope) Under clusterScope: false the chart rendered only a namespace-scoped Role in the release namespace. But the resource-monitor's code: * calls core_v1_api.list_pod_for_all_namespaces(field_selector=spec.nodeName=...) -- a CLUSTER-SCOPED list verb a namespaced Role can never satisfy; and * read_namespaced_pod()s its OWN pod, which lives in .Values.nodeAgents.namespace.name (NOT .Release.Namespace). So with clusterScope: false the DaemonSet 403'd on startup and crashlooped (70+ restarts observed on a live cluster). Per-node monitoring is intrinsically cluster-scoped. Always render the read-only ClusterRole + ClusterRoleBinding regardless of clusterScope (get/list/watch on pods/nodes/namespaces + metrics; no write, exec, or secret access). resourceMonitor: false still fully disables the component. clusterScope continues to gate the training/jobs isolation footprint elsewhere -- it must not leave the node monitor without permissions it cannot run without. Co-Authored-By: Claude Opus 4.8 --- client/templates/resource-monitor-rbac.yaml | 64 ++++++--------------- 1 file changed, 18 insertions(+), 46 deletions(-) diff --git a/client/templates/resource-monitor-rbac.yaml b/client/templates/resource-monitor-rbac.yaml index af59347..59ead14 100644 --- a/client/templates/resource-monitor-rbac.yaml +++ b/client/templates/resource-monitor-rbac.yaml @@ -11,7 +11,24 @@ metadata: labels: {{- include "tracebloc.labels" . | nindent 4 }} app: {{ include "tracebloc.resourceMonitorName" . }} -{{- if ne .Values.clusterScope false }} +{{/* + Per-node monitoring is INTRINSICALLY cluster-scoped, so the resource-monitor + always needs a ClusterRole -- it is deliberately NOT gated on .Values.clusterScope. + The code path that requires it: + * resource_monitor.py uses core_v1_api.list_pod_for_all_namespaces( + field_selector="spec.nodeName=") to enumerate every pod on the node. + list_pod_for_all_namespaces is a CLUSTER-SCOPED list verb that a namespaced + Role can never satisfy (it 403s -> CrashLoopBackOff). + * It also read_namespaced_pod()s its OWN pod, which lives in + .Values.nodeAgents.namespace.name (NOT .Release.Namespace), so a Role scoped + to the release namespace would miss it too. + This ClusterRole is strictly READ-ONLY (get/list/watch on pod/node metadata + + metrics); it grants no write/exec/secret access. clusterScope continues to gate + the training/jobs isolation footprint elsewhere -- it must not cripple node + telemetry by leaving the DaemonSet without the permissions it cannot run without. + If a deployment genuinely cannot allow any cluster-scoped read, disable the + monitor entirely via .Values.resourceMonitor=false rather than deploying it broken. +*/}} --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole @@ -49,49 +66,4 @@ subjects: - kind: ServiceAccount name: {{ include "tracebloc.resourceMonitorName" . }} namespace: {{ .Values.nodeAgents.namespace.name }} -{{- else }} ---- -# Role + RoleBinding live in the RELEASE namespace so the resource-monitor -# can list pods/logs where the actual workloads run. The RoleBinding -# subject points at the ServiceAccount in the node-agents namespace -# (cross-namespace bindings are valid; the Role scope is what matters). -apiVersion: rbac.authorization.k8s.io/v1 -kind: Role -metadata: - name: tracebloc-resource-monitor-{{ .Release.Name }} - namespace: {{ .Release.Namespace }} - annotations: - meta.helm.sh/release-name: {{ .Release.Name }} - meta.helm.sh/release-namespace: {{ .Release.Namespace }} - labels: - {{- include "tracebloc.labels" . | nindent 4 }} - app: {{ include "tracebloc.resourceMonitorName" . }} -rules: - - apiGroups: [""] - resources: ["pods", "pods/log"] - verbs: ["get", "list", "watch"] - - apiGroups: ["metrics.k8s.io"] - resources: ["pods"] - verbs: ["get", "list", "watch"] ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - name: tracebloc-resource-monitor-{{ .Release.Name }} - namespace: {{ .Release.Namespace }} - annotations: - meta.helm.sh/release-name: {{ .Release.Name }} - meta.helm.sh/release-namespace: {{ .Release.Namespace }} - labels: - {{- include "tracebloc.labels" . | nindent 4 }} - app: {{ include "tracebloc.resourceMonitorName" . }} -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: tracebloc-resource-monitor-{{ .Release.Name }} -subjects: - - kind: ServiceAccount - name: {{ include "tracebloc.resourceMonitorName" . }} - namespace: {{ .Values.nodeAgents.namespace.name }} -{{- end }} {{- end }} From 6a869d831cd148c06e53d814e7eeb4f38ab81f44 Mon Sep 17 00:00:00 2001 From: shujaat hasan Date: Tue, 2 Jun 2026 14:36:18 +0200 Subject: [PATCH 2/9] test(resource-monitor): assert always-cluster-scoped RBAC under clusterScope=false Follow-up to the RBAC fix: node_agents_namespace_test.yaml still asserted the old behavior (namespaced Role + RoleBinding in the release namespace when clusterScope=false). Update that case to assert the corrected contract -- a ClusterRole + ClusterRoleBinding always render (with no metadata.namespace), while the subject SA still lives in the node-agents namespace. The clusterScope=false path stays under test; only the asserted behavior changes to match the fix. Verified with `helm unittest` (all resource-monitor suites pass). Co-Authored-By: Claude Opus 4.8 --- client/tests/node_agents_namespace_test.yaml | 22 +++++++++++--------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/client/tests/node_agents_namespace_test.yaml b/client/tests/node_agents_namespace_test.yaml index 67d4214..f597a24 100644 --- a/client/tests/node_agents_namespace_test.yaml +++ b/client/tests/node_agents_namespace_test.yaml @@ -154,28 +154,30 @@ tests: path: subjects[0].namespace value: tracebloc-node-agents - - it: should keep namespace-scoped Role + RoleBinding in the release namespace when clusterScope is false, with SA subject in node-agents + - it: should still render a cluster-scoped ClusterRole + ClusterRoleBinding when clusterScope is false, with SA subject in node-agents template: templates/resource-monitor-rbac.yaml set: clusterScope: false asserts: - # Role must grant access where the monitored workloads live (release ns) + # Per-node monitoring relies on list_pod_for_all_namespaces (a cluster-scoped + # verb a namespaced Role can never satisfy) and reads its own pod in the + # node-agents namespace, so resource-monitor RBAC is intentionally decoupled + # from clusterScope -- it is ALWAYS cluster-scoped. clusterScope still gates + # the training/jobs isolation footprint elsewhere. - isKind: - of: Role + of: ClusterRole documentIndex: 1 - - equal: + # Cluster-scoped resources carry no metadata.namespace + - isNull: path: metadata.namespace - value: tracebloc-templates documentIndex: 1 - # RoleBinding sits in the release ns so it applies the Role there - isKind: - of: RoleBinding + of: ClusterRoleBinding documentIndex: 2 - - equal: + - isNull: path: metadata.namespace - value: tracebloc-templates documentIndex: 2 - # ...but the subject SA lives in the node-agents namespace + # ...but the subject SA still lives in the node-agents namespace - equal: path: subjects[0].namespace value: tracebloc-node-agents From 1070ca1464e13befc51e3a82e1b127ab1109f578 Mon Sep 17 00:00:00 2001 From: shujaat hasan Date: Tue, 2 Jun 2026 16:20:17 +0200 Subject: [PATCH 3/9] fix(rbac): grant `get` on configmaps/secrets to jobs-manager SA MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The ingestion endpoint's orphan-resource verify path (client-runtime#52) and missing-row self-heal (client-runtime#54) read the existing ConfigMap/Secret on a create-409 to confirm content matches before reuse. The Role/ClusterRole only granted `create`, so those reads returned Forbidden and the endpoint 500'd instead of the intended 409/200-replay — verified live on the dev cluster. Add `get` alongside `create` in both the ClusterRole (clusterScope: true) and namespace Role (clusterScope: false) branches. Co-Authored-By: Claude Opus 4.8 --- client/templates/rbac.yaml | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/client/templates/rbac.yaml b/client/templates/rbac.yaml index 3982a5c..9f38644 100644 --- a/client/templates/rbac.yaml +++ b/client/templates/rbac.yaml @@ -44,7 +44,12 @@ rules: verbs: ["create"] - apiGroups: [""] resources: ["configmaps", "secrets"] - verbs: ["create"] + # `get` is required by jobs-manager's orphan-resource verify path + # (client-runtime#52) and the missing-row self-heal (client-runtime#54): + # on a create-409 it reads the existing ConfigMap/Secret to confirm the + # content matches before reusing it. Without `get`, those reads return + # Forbidden and the endpoint 500s instead of the intended 409/replay. + verbs: ["create", "get"] --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding @@ -95,7 +100,12 @@ rules: # to authenticate ingestion calls — see the ClusterRole branch above. - apiGroups: [""] resources: ["configmaps", "secrets"] - verbs: ["create"] + # `get` is required by jobs-manager's orphan-resource verify path + # (client-runtime#52) and the missing-row self-heal (client-runtime#54): + # on a create-409 it reads the existing ConfigMap/Secret to confirm the + # content matches before reusing it. Without `get`, those reads return + # Forbidden and the endpoint 500s instead of the intended 409/replay. + verbs: ["create", "get"] --- apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding From 5f4e4d6d8caf1b9a996268869c07c707062b9469 Mon Sep 17 00:00:00 2001 From: lukasWuttke <54042461+LukasWodka@users.noreply.github.com> Date: Wed, 3 Jun 2026 09:54:20 +0200 Subject: [PATCH 4/9] ci(helm): guard that the pinned ingestor digest is multi-arch (closes #186) (#187) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add a helm-ci job (ingestor-multiarch) that parses images.ingestor.digest and fails the build unless it's a multi-arch index (linux/amd64 + linux/arm64). Greenfield installs spawn the ingestor Job from this PINNED digest before image-refresh first ticks, so an amd64-only pin breaks data ingestion on arm64 hosts (Apple Silicon, Graviton) with "no match for platform" / ImagePullBackOff. This would have caught #160 (the amd64-only v0.3.1 pin) before it shipped. ghcr.io/tracebloc/ingestor is public -> no secrets. Verified: passes on the current multi-arch baseline (sha256:d361fa77, v0.3.2 / #184), fails on the old amd64-only sha256:a0861ea9. Note: the digest is already multi-arch on develop as of v0.3.2 (#184 — the same d361fa77 index this PR previously bumped to), so #187 no longer touches values.yaml; it adds only the regression guard so an amd64-only pin can't slip back in. Co-authored-by: Claude Opus 4.8 --- .github/workflows/helm-ci.yaml | 24 ++++++++++++++++++++++++ client/values.yaml | 7 +++++++ 2 files changed, 31 insertions(+) diff --git a/.github/workflows/helm-ci.yaml b/.github/workflows/helm-ci.yaml index 9abefb0..8f6195c 100644 --- a/.github/workflows/helm-ci.yaml +++ b/.github/workflows/helm-ci.yaml @@ -116,6 +116,30 @@ jobs: > /dev/null echo "Schema validation passed for ${{ matrix.platform }}" + ingestor-multiarch: + # Guard: the chart's PINNED ingestor digest must be a multi-arch index + # (linux/amd64 + linux/arm64). Greenfield installs spawn the ingestor from this + # pinned digest (before image-refresh ticks), so an amd64-only pin breaks data + # ingestion on arm64 hosts (Apple Silicon, Graviton) with ImagePullBackOff. This + # would have caught #160 (which pinned the amd64-only v0.3.1). See client#186. + name: Pinned ingestor digest is multi-arch + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Assert images.ingestor.digest supports linux/amd64 + linux/arm64 + run: | + digest=$(yq '.images.ingestor.digest' client/values.yaml) + echo "Pinned ingestor digest: $digest" + if [ -z "$digest" ] || [ "$digest" = "null" ]; then + echo "::error::images.ingestor.digest is empty — it must be a pinned multi-arch digest."; exit 1 + fi + plats=$(docker buildx imagetools inspect "ghcr.io/tracebloc/ingestor@$digest" 2>&1 \ + | awk '/Platform:/{print $2}' | grep -v '^unknown' | sort -u) + echo "Platforms: $(echo "$plats" | paste -sd' ' -)" + echo "$plats" | grep -qx 'linux/arm64' || { echo "::error::Pinned ingestor digest is NOT multi-arch (no linux/arm64). arm64 installs (Apple Silicon, Graviton) would fail ingestion with 'no match for platform' / ImagePullBackOff. Pin a multi-arch :0.x index. See client#186 / #160."; exit 1; } + echo "$plats" | grep -qx 'linux/amd64' || { echo "::error::Pinned ingestor digest is missing linux/amd64."; exit 1; } + echo "OK — pinned ingestor digest is multi-arch (amd64 + arm64)." + # Installer script tests (bats + Pester) + the cross-distro prerequisite matrix # live in their own workflow: .github/workflows/installer-tests.yaml # (triggered on scripts/** changes). diff --git a/client/values.yaml b/client/values.yaml index b550d70..4cf25c8 100644 --- a/client/values.yaml +++ b/client/values.yaml @@ -220,6 +220,13 @@ images: # first multi-arch image (amd64 + arm64, #24) so arm64 nodes can run # the ingestor. Bump only when greenfield installs should start on a # different version; ongoing rollouts are managed by image-refresh. + # + # This digest MUST be a multi-arch index (linux/amd64 + linux/arm64). + # jobs-manager spawns the ingestor Job by this pinned digest, so an + # amd64-only pin breaks ingestion on arm64 hosts (Apple Silicon, + # Graviton) with "no match for platform" / ImagePullBackOff (#186; the + # amd64-only v0.3.1 pin in #160 was the regression). Enforced by the + # `ingestor-multiarch` job in .github/workflows/helm-ci.yaml. digest: "sha256:d361fa778554ab171adcc2671eaf109c32f3d2af339c7920a2d38d102ce53678" # Floating tag polled by image-refresh. The team's ghcr.io # publishing convention uses semver-style float tags — `0` tracks From d970f5fc69c86118aa1de073b0920c16c754f5d1 Mon Sep 17 00:00:00 2001 From: "Asad Iqbal (Saadi)" Date: Thu, 4 Jun 2026 11:17:46 +0500 Subject: [PATCH 5/9] fix(#190): fail image-refresh loudly when the ingestor (ghcr) digest can't resolve (#191) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit image-refresh silently skipped every tick when get_latest_digest returned empty for the ghcr.io ingestor image (egress/proxy/firewall to ghcr.io, or a blocked token endpoint) — never reaching the registry-drift branch that sets the new digest. jobs-manager + pods-monitor pull from docker.io and refreshed fine, so the CronJob looked healthy while the ingestor digest stayed pinned on the install-time baseline. That's why the berlin-team arm64 install sat on the amd64-only v0.3.1 digest even after :0.3 went multi-arch (#186 follow-up #2). Now count consecutive ingestor-resolve failures on a deployment annotation: - below imageRefresh.ingestorResolveFailureThreshold (default 3, ~45 min at the 15-min schedule) -> WARN + skip, as before (tolerate transient blips); - at/above it -> ERROR with actionable guidance, a tracebloc.io/ingestor-refresh-last-error annotation, and a non-zero exit so the Job fails visibly in `kubectl get cronjob` / monitoring — the same surfacing idiom Pass 2's stuck-rollout check already relies on; - a successful resolve clears the streak. Threshold is nil-guarded (default 3) for --reuse-values upgrades and schema-validated (integer >= 1). The digest-resolution logic itself is unchanged (verified correct: it returns the multi-arch index digest). helm unittest 146/146, helm lint clean, shellcheck + sh -n clean. Co-authored-by: Claude Opus 4.8 --- client/templates/image-refresh-cronjob.yaml | 56 ++++++++++++++++++++- client/tests/image_refresh_test.yaml | 50 ++++++++++++++++++ client/values.schema.json | 6 +++ client/values.yaml | 11 ++++ 4 files changed, 122 insertions(+), 1 deletion(-) diff --git a/client/templates/image-refresh-cronjob.yaml b/client/templates/image-refresh-cronjob.yaml index ee92ed5..af4e2e3 100644 --- a/client/templates/image-refresh-cronjob.yaml +++ b/client/templates/image-refresh-cronjob.yaml @@ -83,6 +83,12 @@ data: log " client images: tracebloc/{jobs-manager,pods-monitor} on docker.io under tag=$IMAGE_TAG" log " ingestor image: tracebloc/ingestor on ghcr.io under tag=$INGESTOR_TAG (pinned=$INGESTOR_PINNED)" + # Consecutive failed ghcr ingestor-digest resolves before image-refresh + # escalates from WARN+skip to a failed Job (#186 #2). The chart injects + # this from imageRefresh.ingestorResolveFailureThreshold; default here + # too so a hand-edited CronJob missing the env var still runs under -u. + : "${INGESTOR_RESOLVE_FAIL_THRESHOLD:=3}" + # Get an anonymous pull-scope token for one repository on a # registry. Both docker.io and ghcr.io support anonymous tokens for # public images; only the issuer URL differs. tr cleanup handles @@ -284,9 +290,50 @@ data: log " ingestor.autoRefresh: false in values; skipping (operator opted into pinning)" else latest_ingestor="$(get_latest_digest "tracebloc/ingestor" "$INGESTOR_TAG" "ghcr.io" || true)" + ingestor_fail_key="tracebloc.io/ingestor-refresh-consecutive-failures" + ingestor_err_key="tracebloc.io/ingestor-refresh-last-error" if [ -z "$latest_ingestor" ]; then - log " WARN: could not resolve latest digest (rate-limited or transient); skipping this tick" + # Could not resolve the ingestor digest from ghcr.io. A transient + # blip (rate-limit, momentary DNS) and a PERSISTENT failure (egress + # to ghcr.io firewalled, the ghcr token endpoint blocked, a proxy + # that allowlists docker.io but not ghcr.io) are indistinguishable + # on a single tick — so COUNT consecutive failures and escalate + # once they cross the threshold, instead of skipping silently + # forever. #186 (#2): berlin-team sat on the amd64-only baseline + # because every ingestor tick hit this branch while the docker.io + # images (jobs-manager, pods-monitor) refreshed fine — the CronJob + # looked healthy and nothing surfaced the ghcr failure. + prev_fails="$(get_annotation "$ingestor_fail_key" || true)" + case "$prev_fails" in ''|*[!0-9]*) prev_fails=0 ;; esac + if [ "$prev_fails" -ge "$INGESTOR_RESOLVE_FAIL_THRESHOLD" ]; then + # Already escalated on an earlier tick; stay loud (fail the Job) + # without inflating the counter — it caps at the threshold. + log " ERROR: still cannot resolve ghcr.io/tracebloc/ingestor:${INGESTOR_TAG} (>= ${INGESTOR_RESOLVE_FAIL_THRESHOLD} consecutive failures). Ingestor digest is NOT auto-refreshing; ingestion may be stuck on a stale image. Check egress to ghcr.io and its token endpoint from this namespace. See client#186." + exit 1 + fi + fails=$((prev_fails + 1)) + kubectl annotate deployment -n "$RELEASE_NAMESPACE" "$DEPLOYMENT_NAME" \ + "${ingestor_fail_key}=${fails}" --overwrite + if [ "$fails" -ge "$INGESTOR_RESOLVE_FAIL_THRESHOLD" ]; then + # Persistent. Record a human-readable last-error and fail the Job + # so it surfaces in `kubectl get cronjob` and monitoring — the + # same "failed Job = operator-visible" idiom Pass 2's stuck- + # rollout check already relies on. + kubectl annotate deployment -n "$RELEASE_NAMESPACE" "$DEPLOYMENT_NAME" \ + "${ingestor_err_key}=could not resolve ghcr.io/tracebloc/ingestor:${INGESTOR_TAG} for ${fails} consecutive ticks" --overwrite + log " ERROR: could not resolve ghcr.io/tracebloc/ingestor:${INGESTOR_TAG} for ${fails} consecutive ticks (threshold ${INGESTOR_RESOLVE_FAIL_THRESHOLD}). Ingestor digest is NOT auto-refreshing; ingestion may be stuck on a stale image (e.g. an amd64-only baseline on arm64 nodes). Check egress to ghcr.io and its token endpoint from this namespace. Failing the Job so this surfaces in monitoring. See client#186." + exit 1 + fi + log " WARN: could not resolve latest ingestor digest (failure ${fails}/${INGESTOR_RESOLVE_FAIL_THRESHOLD}; transient?); skipping this tick" else + # Resolved OK. Clear any prior failure streak so a recovered + # registry/egress blip doesn't leave a stale failure annotation + # (and a future failure starts counting from zero again). + if [ -n "$(get_annotation "$ingestor_fail_key" || true)" ]; then + log " ingestor digest resolved; clearing prior failure streak" + kubectl annotate deployment -n "$RELEASE_NAMESPACE" "$DEPLOYMENT_NAME" \ + "${ingestor_fail_key}-" "${ingestor_err_key}-" >/dev/null 2>&1 || true + fi recorded_ingestor="$(get_annotation "tracebloc.io/last-refreshed-ingestor-digest" || true)" # Always read spec env too — the annotation alone isn't enough # because external actors can revert the spec without touching @@ -484,6 +531,13 @@ spec: # semver-style float tags. Caught in PR #162 review. - name: INGESTOR_TAG value: {{ (default dict (default dict .Values.images).ingestor).tag | default "0.3" | quote }} + # Consecutive failed ghcr ingestor-digest resolves before + # image-refresh stops silently skipping and fails the Job + # loudly (#186 #2). nil-guarded with a `default 3` fallback + # so --reuse-values upgrades from pre-this-PR stored + # manifests (which lack the key) still render. + - name: INGESTOR_RESOLVE_FAIL_THRESHOLD + value: {{ (default dict .Values.imageRefresh).ingestorResolveFailureThreshold | default 3 | quote }} securityContext: allowPrivilegeEscalation: false readOnlyRootFilesystem: true diff --git a/client/tests/image_refresh_test.yaml b/client/tests/image_refresh_test.yaml index a13c665..42f458d 100644 --- a/client/tests/image_refresh_test.yaml +++ b/client/tests/image_refresh_test.yaml @@ -463,6 +463,56 @@ tests: name: INGESTOR_TAG value: staging + - it: ingestor INGESTOR_RESOLVE_FAIL_THRESHOLD defaults to "3" (#186 #2) + # Below this many consecutive ghcr resolve failures, image-refresh + # WARNs + skips; at/above it the script fails the Job loudly instead + # of silently leaving jobs-manager on a stale digest. + template: templates/image-refresh-cronjob.yaml + documentIndex: 1 + asserts: + - contains: + path: spec.jobTemplate.spec.template.spec.containers[0].env + content: + name: INGESTOR_RESOLVE_FAIL_THRESHOLD + value: "3" + + - it: ingestor INGESTOR_RESOLVE_FAIL_THRESHOLD falls back to "3" when imageRefresh key is absent + # --reuse-values upgrade from a pre-this-PR stored manifest lacks the + # key; the template's `| default 3` must still render a usable value. + template: templates/image-refresh-cronjob.yaml + documentIndex: 1 + set: + imageRefresh: + ingestorResolveFailureThreshold: null + asserts: + - contains: + path: spec.jobTemplate.spec.template.spec.containers[0].env + content: + name: INGESTOR_RESOLVE_FAIL_THRESHOLD + value: "3" + + - it: ingestor INGESTOR_RESOLVE_FAIL_THRESHOLD is overridable + template: templates/image-refresh-cronjob.yaml + documentIndex: 1 + set: + imageRefresh: + ingestorResolveFailureThreshold: 5 + asserts: + - contains: + path: spec.jobTemplate.spec.template.spec.containers[0].env + content: + name: INGESTOR_RESOLVE_FAIL_THRESHOLD + value: "5" + + - it: schema rejects ingestorResolveFailureThreshold below 1 + template: templates/image-refresh-cronjob.yaml + set: + imageRefresh: + ingestorResolveFailureThreshold: 0 + asserts: + - failedTemplate: + errorPattern: "must not be valid against schema|Must be greater than or equal to 1" + - it: schema rejects ingestor.tag=latest template: templates/image-refresh-cronjob.yaml set: diff --git a/client/values.schema.json b/client/values.schema.json index 937b363..9ba0564 100644 --- a/client/values.schema.json +++ b/client/values.schema.json @@ -585,6 +585,12 @@ "pattern": "^[0-9]+(s|m|h)$", "description": "Passed to `kubectl rollout status --timeout`. Allow headroom for bare-metal image pull." }, + "ingestorResolveFailureThreshold": { + "type": "integer", + "minimum": 1, + "default": 3, + "description": "Consecutive failed ghcr.io ingestor-digest resolves before image-refresh fails the Job loudly instead of silently skipping. See #186 (#2). Higher tolerates flakier egress; very high keeps the old skip-forever behaviour." + }, "suspend": { "type": "boolean", "default": false, diff --git a/client/values.yaml b/client/values.yaml index 4cf25c8..5e5eda9 100644 --- a/client/values.yaml +++ b/client/values.yaml @@ -489,6 +489,17 @@ imageRefresh: # on bare-metal first-boot can take minutes, and the CronJob slot # shouldn't hold all night if a rollout genuinely wedges. rolloutTimeout: "10m" + # Consecutive failed ghcr.io ingestor-digest resolves before image-refresh + # stops silently skipping and fails the Job loudly (visible in + # `kubectl get cronjob` / monitoring, plus a `tracebloc.io/ingestor-refresh- + # last-error` annotation on the jobs-manager deployment). Default 3 ≈ 45 min + # at the 15-min schedule: long enough to ride out a transient registry blip, + # short enough to surface a persistent egress/proxy failure (e.g. a cluster + # that reaches docker.io but not ghcr.io — the #186 (#2) root cause). A + # failed resolve BELOW this count still just WARNs and skips the tick. + # Raise it for flakier egress; set it very high to keep the old + # skip-forever behaviour. Only affects the ingestor (ghcr) image. + ingestorResolveFailureThreshold: 3 # CronJob spec knobs. Override per-cluster if needed. suspend: false # Lower history limits than autoUpgrade — this Job runs ~96x/day, the From 18098722963b412fa537e5cd15f692439ce7c01c Mon Sep 17 00:00:00 2001 From: "Asad Iqbal (Saadi)" Date: Thu, 4 Jun 2026 11:30:39 +0500 Subject: [PATCH 6/9] test(requests-proxy): add helm-unittest coverage for requests-proxy Deployment (#194) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit requests-proxy-deployment.yaml was the only data-plane workload template without a unit test. This suite pins the properties most costly to regress: - security-context invariants (no SA-token automount, runAsNonRoot, seccomp RuntimeDefault, runAsUser 1001, no privilege escalation, drop ALL caps, read-only root filesystem) — see docs/SECURITY.md - the single-replica / single gunicorn worker constraint (the pod token registry is process-local; >1 worker silently shards token lookups) - the docker.io/tracebloc/jobs-manager image source and port 8888 - the nil-guarded resource defaults, plus an override case that exercises the default-through-dict fallthrough (guards the historic `readOnlyRootFilesystem: trueresources:` newline-eating regression) Co-authored-by: Claude Opus 4.8 --- client/tests/requests_proxy_test.yaml | 120 ++++++++++++++++++++++++++ 1 file changed, 120 insertions(+) create mode 100644 client/tests/requests_proxy_test.yaml diff --git a/client/tests/requests_proxy_test.yaml b/client/tests/requests_proxy_test.yaml new file mode 100644 index 0000000..3c5f3f8 --- /dev/null +++ b/client/tests/requests_proxy_test.yaml @@ -0,0 +1,120 @@ +suite: Requests Proxy Deployment +# requests-proxy-deployment.yaml had no unit test. It is a workload pod that +# runs in the data plane, so the security-context invariants (see +# docs/SECURITY.md) and the single-worker constraint (the proxy keeps the pod +# token registry in process-local memory — more than one worker silently +# breaks token lookups) need a regression guard. The resources block is also +# nil-guarded against `helm upgrade --reuse-values` from pre-1.3.6 releases; +# pinning the rendered defaults catches a guard that eats the preceding +# newline (the historic `readOnlyRootFilesystem: trueresources:` bug). +templates: + - templates/requests-proxy-deployment.yaml +set: + clientId: "test-id" + clientPassword: "test" + dockerRegistry: + server: https://index.docker.io/v1/ + username: test + password: test + email: test@test.com +tests: + - it: should create the requests-proxy Deployment + asserts: + - isKind: + of: Deployment + - equal: + path: metadata.name + value: RELEASE-NAME-requests-proxy + + - it: should carry standard chart labels + asserts: + - exists: + path: metadata.labels["app.kubernetes.io/name"] + - exists: + path: metadata.labels["helm.sh/chart"] + + - it: should run exactly one replica + # Token registry is process-local; replicas>1 would shard token lookups. + asserts: + - equal: + path: spec.replicas + value: 1 + + - it: should not automount the service account token + asserts: + - equal: + path: spec.template.spec.automountServiceAccountToken + value: false + + - it: should enforce a hardened pod security context + asserts: + - equal: + path: spec.template.spec.securityContext.runAsNonRoot + value: true + - equal: + path: spec.template.spec.securityContext.seccompProfile.type + value: RuntimeDefault + + - it: should enforce a hardened container security context + asserts: + - equal: + path: spec.template.spec.containers[0].securityContext.runAsNonRoot + value: true + - equal: + path: spec.template.spec.containers[0].securityContext.runAsUser + value: 1001 + - equal: + path: spec.template.spec.containers[0].securityContext.allowPrivilegeEscalation + value: false + - equal: + path: spec.template.spec.containers[0].securityContext.readOnlyRootFilesystem + value: true + - contains: + path: spec.template.spec.containers[0].securityContext.capabilities.drop + content: "ALL" + + - it: should pull the jobs-manager image from docker.io + asserts: + - matchRegex: + path: spec.template.spec.containers[0].image + pattern: "^docker\\.io/tracebloc/jobs-manager[:@]" + + - it: should expose the proxy on container port 8888 + asserts: + - equal: + path: spec.template.spec.containers[0].ports[0].containerPort + value: 8888 + + - it: should run gunicorn with a single worker + # Mirrors the replica constraint at the process level — the token registry + # lives in one worker's memory. + asserts: + - contains: + path: spec.template.spec.containers[0].args + content: "--workers=1" + + - it: should render the default resource requests and limits + asserts: + - equal: + path: spec.template.spec.containers[0].resources.requests.cpu + value: 100m + - equal: + path: spec.template.spec.containers[0].resources.requests.memory + value: 256Mi + - equal: + path: spec.template.spec.containers[0].resources.limits.cpu + value: 1000m + - equal: + path: spec.template.spec.containers[0].resources.limits.memory + value: 512Mi + + - it: should honor a resource override through the nil-guard + set: + resources: + requestsProxy: + limits: + memory: 1Gi + asserts: + - equal: + path: spec.template.spec.containers[0].resources.limits.memory + value: 1Gi From 44fe908c208211c4c8ee49a33381fa2a140a68d4 Mon Sep 17 00:00:00 2001 From: "Asad Iqbal (Saadi)" Date: Thu, 4 Jun 2026 14:30:00 +0500 Subject: [PATCH 7/9] fix(#196): allow training-pod egress to the requests-proxy (8888) (#197) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The training-egress NetworkPolicy denies all pod-to-pod / ClusterIP egress (rule 2 excepts the cluster CIDRs) and re-permits only MySQL (rule 3). When the requests-proxy architecture shipped — training pods POST epoch results / FLOPs to requests-proxy-service:8888 instead of holding Service Bus credentials — this template was never updated to re-permit egress to the proxy. Result on every install with the policy enabled: pods hit "requests-proxy-service:8888 ... [Errno 111] Connection refused" at the first epoch finalize → CrashLoopBackOff → all experiments fail. Add rule 4 mirroring the MySQL rule: TCP/8888 to podSelector app=requests-proxy (same namespace). Service selector + port from templates/requests-proxy-service.yaml. Verified: `helm template -f ci/bm-values.yaml --show-only templates/network-policy-training.yaml` renders the new rule as valid YAML. Found live on a fresh client (tracebloc-amazon / k3d): jobs-manager reached the proxy (HTTP 401) while training pods got connection-refused — the only differentiator was this egress policy. Interim: live-patched the cluster + suspended its auto-upgrade CronJob (so reuse-values wouldn't revert the patch); re-enable once this lands + releases. Closes #196. Co-authored-by: Claude Opus 4.8 (1M context) --- client/templates/network-policy-training.yaml | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/client/templates/network-policy-training.yaml b/client/templates/network-policy-training.yaml index c0f0fd1..643d96a 100644 --- a/client/templates/network-policy-training.yaml +++ b/client/templates/network-policy-training.yaml @@ -85,4 +85,19 @@ spec: ports: - port: 3306 protocol: TCP + # 4. requests-proxy — training pods POST epoch results / FLOPs to the + # in-namespace requests-proxy on 8888 (so they never hold Service Bus + # credentials). Rule 2 blocks this ClusterIP egress, so re-permit it + # explicitly, exactly like MySQL above. Without this rule every + # experiment CrashLoopBackOffs at the first epoch finalize with + # "requests-proxy-service:8888 ... Connection refused" (client#196). + # Service selector + port: templates/requests-proxy-service.yaml + # (app=requests-proxy, 8888). + - to: + - podSelector: + matchLabels: + app: requests-proxy + ports: + - port: 8888 + protocol: TCP {{- end }} From 36b78e4a3089548b23ef5dfaf06ffff2f73dba86 Mon Sep 17 00:00:00 2001 From: lukasWuttke <54042461+LukasWodka@users.noreply.github.com> Date: Thu, 4 Jun 2026 11:30:18 +0200 Subject: [PATCH 8/9] Installer UX: drop PriorityClass, fix namespace, one-per-machine guard, surface version (#192) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(chart): drop the data-plane PriorityClass by default The cluster-scoped, fixed-name `tracebloc-data-plane` PriorityClass was the only thing forcing one tracebloc client per cluster (a second release collided on it with a cryptic Helm error) and blocking multiple tracebloc namespaces in one BYO cluster. mysql doesn't need it: memory requests==limits (last evicted under memory pressure), data on a PVC (eviction = transient restart, not data loss), and a PDB guards voluntary disruptions. Its only unique benefit was letting the scheduler preempt training jobs to keep mysql scheduled on a packed node — a narrow case. Default priorityClass.create=false + name="" so new installs template no PriorityClass and mysql carries no priorityClassName. Opt back in (create:true + name) on contended clusters, or reference an out-of-band one (create:false + name:). helm-unittest updated; 144/144 pass. Co-Authored-By: Claude Opus 4.8 * feat(installer): fixed namespace + one-per-machine guard; drop workspace prompt The "Choose a workspace name" prompt asked the user to invent a label that isn't their identity (the backend identifies a client by its credentials, not this string — it's just the local k8s namespace / Helm release name; the installer even discards the auth response body). It defaulted to a meaningless "default" and was the field that collided on a second install. - Drop the prompt; TB_NAMESPACE defaults to a fixed "tracebloc" (env-overridable for advanced/GitOps setups). - One-client-per-machine guard: after credentials verify, compare the entered Client ID against any client already installed here (helm get values). Same ID = a normal re-run/upgrade; a DIFFERENT ID hard-blocks with an explanation and options (repair / switch via `k3d cluster delete` / use another machine) instead of silently re-pointing the machine. This replaces the accidental PriorityClass collision (now dropped) with an intentional, explained guard. - Document the TB_NAMESPACE override; update bats (input sequences + 2 new guard tests). bats 26/27 — the 1 failure is a pre-existing macOS-bash-3.2 quirk in _extract_yaml_value, unrelated (CI bash 5 passes it). NOTE: install-k8s.ps1 + its Pester tests still need the same mirror (follow-up). Co-Authored-By: Claude Opus 4.8 * feat(installer-ps): mirror fixed namespace + one-per-machine guard (PowerShell) Mirrors the bash change in install-k8s.ps1: - drop the "Choose a workspace name" prompt; TB_NAMESPACE defaults to a fixed "tracebloc" (override via $env:TB_NAMESPACE). - one-client-per-machine guard: after credentials verify, compare the entered Client ID against any client already installed here (helm get values); a different ID hard-blocks with the same explanation/options as bash. - Pester: 2 new guard tests (block-different / allow-same). The existing Install-ClientHelm tests use dispatch-by-prompt Read-Host mocks, so the prompt removal doesn't disturb them. No pwsh locally -> verified via CI (Pester ubuntu+windows + PSScriptAnalyzer). Co-Authored-By: Claude Opus 4.8 * fix(installer): scan all namespaces in the one-per-machine guard The guard checked only the `tracebloc` namespace, so a client installed by an older installer version (default namespace `default`, or a custom name) wasn't detected -- a re-run could create a second coexisting client. Now enumerate all client-chart releases (helm list -A) and compare each one's clientId, covering both fresh and migrated installs. bash uses jq (already a dependency; falls back to the tracebloc namespace if absent); PowerShell uses ConvertFrom-Json. The block message names the namespace. bats + Pester guard tests updated. Verified: bats green locally (jq path); PowerShell via CI Pester. Co-Authored-By: Claude Opus 4.8 * feat(installer): show client (chart) version in summary + --diagnose Users had no easy way to see which client version they're on (the CLI isn't shipped yet; `helm list` needs the namespace, and nothing surfaced it). Show the chart version where they already look: - install summary: a "Version" line next to Workspace. - --diagnose: as the first console line + recorded in the bundle header (the #1 thing support needs). Adds a best-effort `_chart_version` / Get-ChartVersion helper (greps helm's CHART column -> no jq). bash + PowerShell; bats + Pester coverage added. Verified: summary.bats + diagnose.bats green locally; ps1 via CI Pester. Co-Authored-By: Claude Opus 4.8 --------- Co-authored-by: Claude Opus 4.8 --- client/tests/mysql_test.yaml | 12 +++- client/tests/priority_class_pdb_test.yaml | 12 +++- client/values.yaml | 29 +++++---- scripts/install-k8s.ps1 | 78 ++++++++++++++++++----- scripts/install-k8s.sh | 2 + scripts/lib/common.sh | 10 +++ scripts/lib/diagnose.sh | 8 ++- scripts/lib/install-client-helm.sh | 62 +++++++++++++----- scripts/lib/summary.sh | 2 + scripts/tests/diagnose.bats | 11 ++++ scripts/tests/install-client-helm.bats | 54 +++++++++++++--- scripts/tests/install-k8s.Tests.ps1 | 40 ++++++++++++ scripts/tests/summary.bats | 8 +++ 13 files changed, 270 insertions(+), 58 deletions(-) diff --git a/client/tests/mysql_test.yaml b/client/tests/mysql_test.yaml index ec30443..ad6398d 100644 --- a/client/tests/mysql_test.yaml +++ b/client/tests/mysql_test.yaml @@ -61,8 +61,18 @@ tests: path: spec.template.spec.containers[0].livenessProbe.failureThreshold value: 5 - - it: should reference the data-plane PriorityClass when enabled + - it: should NOT set priorityClassName by default (PriorityClass dropped) template: templates/mysql-deployment.yaml + asserts: + - notExists: + path: spec.template.spec.priorityClassName + + - it: should reference the PriorityClass when a name is set + template: templates/mysql-deployment.yaml + set: + priorityClass: + create: true + name: tracebloc-data-plane asserts: - equal: path: spec.template.spec.priorityClassName diff --git a/client/tests/priority_class_pdb_test.yaml b/client/tests/priority_class_pdb_test.yaml index 7a53699..33c4b56 100644 --- a/client/tests/priority_class_pdb_test.yaml +++ b/client/tests/priority_class_pdb_test.yaml @@ -7,8 +7,18 @@ set: clientId: "test-id" clientPassword: "test" tests: - - it: should create the data-plane PriorityClass by default + - it: should NOT render the PriorityClass by default (dropped) template: templates/priority-class.yaml + asserts: + - hasDocuments: + count: 0 + + - it: should render the PriorityClass when explicitly enabled + template: templates/priority-class.yaml + set: + priorityClass: + create: true + name: tracebloc-data-plane asserts: - isKind: of: PriorityClass diff --git a/client/values.yaml b/client/values.yaml index 5e5eda9..5dbccd1 100644 --- a/client/values.yaml +++ b/client/values.yaml @@ -313,22 +313,23 @@ resources: memory: "512Mi" # -- PriorityClass for the data-plane (mysql). -# Cluster-scoped resource. Created with helm.sh/resource-policy: keep so a -# release uninstall does not yank it from sibling releases that share it. -# Value 1,000,000 sits well above default (0) so the scheduler will preempt -# noisy training jobs to keep mysql scheduled, but well below -# system-cluster-critical (2,000,000,000). +# DROPPED BY DEFAULT (create: false, name: ""). mysql doesn't need it: its +# memory requests==limits (so it's among the last evicted under memory +# pressure), its data lives on a PVC (eviction = a transient restart, never +# data loss), and a PodDisruptionBudget guards voluntary disruptions. Dropping +# this cluster-scoped, fixed-name object also lets multiple tracebloc +# namespaces coexist in one cluster (BYO) and removes the one-client-per- +# cluster install collision. # -# create: false — chart does not template the PriorityClass; you (or your -# GitOps tool / shared platform) manage it out-of-band. -# The mysql pod still references `name`, so make sure the -# PriorityClass exists at install time. -# name: "" — disable the priorityClassName reference entirely. mysql -# falls back to the cluster default priority (0) and loses -# the OOM-protection this chart's mysql tuning relies on. +# Opt back in ONLY on a heavily-contended cluster where you want the scheduler +# to preempt noisy training jobs to keep mysql scheduled: +# create: true, name: , value: ~1000000 (above default 0, below +# system-cluster-critical 2,000,000,000). +# For a PriorityClass your platform manages out-of-band, use create: false + +# name: (mysql references it without the chart templating it). priorityClass: - create: true - name: tracebloc-data-plane + create: false + name: "" value: 1000000 # -- PodDisruptionBudgets. diff --git a/scripts/install-k8s.ps1 b/scripts/install-k8s.ps1 index 67dcfbb..c890bea 100644 --- a/scripts/install-k8s.ps1 +++ b/scripts/install-k8s.ps1 @@ -112,6 +112,15 @@ function ConvertTo-WorkspaceName { return $sanitized } +# Best-effort chart version of the installed client release (e.g. "1.4.4"); +# empty if not found / cluster unreachable. Greps helm's CHART column. +function Get-ChartVersion { + param([string]$Namespace = "tracebloc") + $out = (helm list -n $Namespace 2>$null) | Out-String + if ($out -match 'client-([0-9][^\s]*)') { return $Matches[1] } + return "" +} + # ============================================================================= # CONFIGURATION # ============================================================================= @@ -1006,7 +1015,6 @@ function Install-ClientHelm { } $valuesFile = Join-Path $HOST_DATA_DIR "values.yaml" - $defaultNamespace = "default" $defaultClientId = "" $defaultClientPassword = "" @@ -1027,21 +1035,15 @@ function Install-ClientHelm { } } - # -- Workspace name prompt -- - PromptHeader "Choose a workspace name" - Hint "This identifies your tracebloc client on this machine." - Write-Host "" - Hint "Examples: myteam, vision-lab, lukas" - Write-Host "" - $nsInput = Read-Host " Workspace name [$defaultNamespace]" - $rawName = if ($nsInput) { $nsInput } else { $defaultNamespace } - $TB_NAMESPACE = ConvertTo-WorkspaceName -Input_ $rawName + # -- Namespace (fixed; not prompted) -- + # The on-prem client is one-per-machine and is identified to the backend by + # its credentials (clientId), not by this name -- so we don't ask the user to + # invent one. It's just the local k8s namespace / Helm release name. + # Advanced / GitOps setups can override with TB_NAMESPACE=. + $rawNs = if ($env:TB_NAMESPACE) { $env:TB_NAMESPACE } else { "tracebloc" } + $TB_NAMESPACE = ConvertTo-WorkspaceName -Input_ $rawNs $script:TB_NAMESPACE = $TB_NAMESPACE # share with Wait-ForClientReady / Print-Summary - if ($TB_NAMESPACE -ne $rawName) { - Info "Using workspace: $TB_NAMESPACE" - } - # -- Step 4/4: Connect to tracebloc network -- Step 4 4 "Connect to tracebloc network" @@ -1101,6 +1103,43 @@ function Install-ClientHelm { $defaultClientId = ""; $defaultClientPassword = "" } + # -- One-client-per-machine guard -- + # A machine runs exactly one tracebloc client: it shares this cluster and the + # host's CPU/RAM/GPU, and the platform counts each client as separate + # capacity. If a DIFFERENT client is already installed here, a re-install + # would silently re-point the machine -- so we stop and let the operator + # decide. The same clientId is a normal re-run/upgrade and passes through. + # Check ANY namespace: a fresh install lands in 'tracebloc', but an install + # from an older installer version may be in a different namespace. Enumerate + # client-chart releases and read each clientId (ConvertFrom-Json -- no jq). + $existingId = ""; $existingNs = "" + $listJson = (helm list -A -o json 2>$null) | Out-String + if ($LASTEXITCODE -eq 0 -and $listJson.Trim()) { + try { + foreach ($rel in ($listJson | ConvertFrom-Json)) { + if ($rel.chart -and $rel.chart.StartsWith("client-")) { + $vals = (helm get values $rel.name -n $rel.namespace 2>$null) | Out-String + if ($vals -match 'clientId:\s*"([^"]+)"') { $existingId = $Matches[1].Trim(); $existingNs = $rel.namespace; break } + } + } + } catch { } + } + if ($existingId -and $existingId -ne $TB_CLIENT_ID) { + Write-Host "" + Warn "This machine already runs the tracebloc client '$existingId' (namespace '$existingNs')." + Hint "tracebloc runs one client per machine -- it shares this cluster and host" + Hint "resources, and the platform counts each client as separate capacity." + Write-Host "" + Hint "You entered a different Client ID ('$TB_CLIENT_ID'). Pick one:" + Hint " - Repair / update '$existingId' -> re-run with that same Client ID" + Hint " - Switch to '$TB_CLIENT_ID' -> remove the current client first:" + Hint " k3d cluster delete $CLUSTER_NAME (wipes this client + its local data)" + Hint " then re-run this installer" + Hint " - Run both clients -> install on a separate machine" + Write-Host "" + Err "Refusing to replace the existing client. See the options above." + } + $passwordEscaped = $TB_CLIENT_PASSWORD -replace "'", "''" $gpuVal = "" @@ -1257,6 +1296,8 @@ function Print-Summary { Write-Host " " -NoNewline; Write-Host "$([char]0x2714) Connected to tracebloc" -ForegroundColor Green Write-Host "" Write-Host " Workspace : " -NoNewline; Write-Host $ns -ForegroundColor Cyan + $cver = Get-ChartVersion -Namespace $ns; if (-not $cver) { $cver = "unknown" } + Write-Host " Version : " -NoNewline; Write-Host $cver -ForegroundColor Cyan Write-Host " Mode : " -NoNewline; Write-Host $mode -ForegroundColor Cyan Write-Host "" Write-Host " Your client is live. Confirm it shows as Online:" @@ -1460,8 +1501,6 @@ function Invoke-DiagnoseBundle { $d = Join-Path $work "tracebloc-diagnose-$ts" New-Item -ItemType Directory -Path (Join-Path $d "logs") -Force | Out-Null - Info "Collecting diagnostics -- this is safe; credentials are redacted before the file is written." - # Namespace discovery (TB_NAMESPACE isn't set on a standalone diagnose run). $ns = $TB_NAMESPACE if (-not $ns) { @@ -1470,9 +1509,14 @@ function Invoke-DiagnoseBundle { } if (-not $ns) { $ns = "default" } + # Surface the client version first -- the #1 thing support needs to know. + $cver = Get-ChartVersion -Namespace $ns; if (-not $cver) { $cver = "unknown" } + Info "tracebloc client version: $cver (namespace: $ns)" + Info "Collecting diagnostics -- this is safe; credentials are redacted before the file is written." + # host / versions $h = @("# tracebloc diagnose ($ts)", "OS: Windows ARCH: $(Get-WindowsArch)", - "CLIENT_ENV: $($env:CLIENT_ENV) CLUSTER_NAME: $cn NAMESPACE: $ns", "## versions", + "CLIENT_ENV: $($env:CLIENT_ENV) CLUSTER_NAME: $cn NAMESPACE: $ns", "CLIENT VERSION: $cver", "## versions", (k3d version 2>&1 | Out-String), (kubectl version --client 2>&1 | Out-String), (helm version --short 2>&1 | Out-String), (docker version 2>&1 | Out-String)) try { $cs = Get-CimInstance Win32_ComputerSystem -ErrorAction Stop; $h += "CPUs=$($cs.NumberOfLogicalProcessors) MemBytes=$($cs.TotalPhysicalMemory)" } catch {} diff --git a/scripts/install-k8s.sh b/scripts/install-k8s.sh index 615a701..804e535 100755 --- a/scripts/install-k8s.sh +++ b/scripts/install-k8s.sh @@ -15,6 +15,8 @@ # # Environment variable overrides (optional): # CLUSTER_NAME=myapp default: tracebloc +# TB_NAMESPACE=myns default: tracebloc (k8s namespace + local label; +# not prompted — the client is identified by its credentials) # SERVERS=1 default: 1 (control-plane nodes) # AGENTS=1 default: 1 (worker nodes) # K8S_VERSION=v1.29.4-k3s1 default: latest stable k3s diff --git a/scripts/lib/common.sh b/scripts/lib/common.sh index 6a5a979..96d08c0 100755 --- a/scripts/lib/common.sh +++ b/scripts/lib/common.sh @@ -35,6 +35,15 @@ hint() { echo -e " ${DIM}$*${RESET}"; } # ── Utility ────────────────────────────────────────────────────────────────── has() { command -v "$1" &>/dev/null; } +# Best-effort chart version of the installed client release in namespace $1 +# (e.g. "1.4.4"); empty if not found / cluster unreachable. Greps helm's CHART +# column ("client-"), so it needs no jq. +_chart_version() { + local ns="${1:-${TB_NAMESPACE:-tracebloc}}" + has helm || return 0 + helm list -n "$ns" 2>/dev/null | grep -oE 'client-[0-9][^[:space:]]*' | head -1 | sed 's/^client-//' +} + # ── macOS: Docker Desktop architecture vs machine (for wrong-arch UX) ──────── # Call early on macOS to fail fast with clear instructions if Docker.app # is for the wrong architecture (e.g. Intel Docker on Apple Silicon). @@ -361,6 +370,7 @@ Commands: Advanced configuration (environment variables): CLUSTER_NAME Cluster name (default: tracebloc) + TB_NAMESPACE Namespace / workspace label (default: tracebloc) SERVERS Control-plane nodes (default: 1) AGENTS Worker nodes (default: 1) K8S_VERSION k3s image tag (default: v1.29.4-k3s1) diff --git a/scripts/lib/diagnose.sh b/scripts/lib/diagnose.sh index ed74adf..f64acee 100644 --- a/scripts/lib/diagnose.sh +++ b/scripts/lib/diagnose.sh @@ -49,8 +49,6 @@ run_diagnose() { if [[ -z "$outdir" || ! -d "$outdir" ]]; then echo " diagnose: cannot create a temp directory" >&2; return 1; fi d="$outdir/tracebloc-diagnose-$ts"; mkdir -p "$d/logs" - info "Collecting diagnostics — this is safe; credentials are redacted before the file is written." - # Namespace discovery — TB_NAMESPACE isn't set on a standalone diagnose run, # so find the namespace of the jobs-manager pod (falls back to "default"). ns="${TB_NAMESPACE:-}" @@ -59,12 +57,18 @@ run_diagnose() { fi [[ -z "$ns" ]] && ns="default" + # Surface the client version first — the #1 thing support needs to know. + local cver; cver="$(_chart_version "$ns")" + info "tracebloc client version: ${cver:-unknown} (namespace: $ns)" + info "Collecting diagnostics — this is safe; credentials are redacted before the file is written." + # ── host / versions ── { echo "# tracebloc diagnose ($ts)" echo "OS: $(uname -s) $(uname -r)" echo "ARCH: $(uname -m)" echo "CLIENT_ENV: ${CLIENT_ENV:-} CLUSTER_NAME: $cn NAMESPACE: $ns" + echo "CLIENT VERSION: ${cver:-unknown}" echo; echo "## versions" has k3d && k3d version has kubectl && kubectl version --client 2>/dev/null diff --git a/scripts/lib/install-client-helm.sh b/scripts/lib/install-client-helm.sh index 2ffd5e2..6cc95bc 100644 --- a/scripts/lib/install-client-helm.sh +++ b/scripts/lib/install-client-helm.sh @@ -153,13 +153,12 @@ install_client_helm() { if [[ -n "${TRACEBLOC_VALUES_FILE:-}" ]]; then [[ -f "$TRACEBLOC_VALUES_FILE" ]] || error "TRACEBLOC_VALUES_FILE not found: $TRACEBLOC_VALUES_FILE" values_file="$TRACEBLOC_VALUES_FILE" - TB_NAMESPACE="${TB_NAMESPACE:-default}" + TB_NAMESPACE="${TB_NAMESPACE:-tracebloc}" info "Dev mode: using caller-provided values file" log "Using values file: $values_file (namespace: $TB_NAMESPACE)" else local use_existing="" - local default_namespace="default" local default_client_id="" local default_client_password="" @@ -179,19 +178,12 @@ install_client_helm() { fi fi - # ── Workspace name prompt ──────────────────────────────────────────────── - prompt_header "Choose a workspace name" - hint "This identifies your tracebloc client on this machine." - echo "" - hint "Examples: berlin-team, vision-lab, ml-mardan" - echo "" - read -r -p " Workspace name [${default_namespace}]: " TB_NAMESPACE_INPUT - local raw_name="${TB_NAMESPACE_INPUT:-$default_namespace}" - TB_NAMESPACE=$(_sanitize_workspace_name "$raw_name") - - if [[ "$TB_NAMESPACE" != "$raw_name" ]]; then - info "Using workspace: ${BOLD}${TB_NAMESPACE}${RESET}" - fi + # ── Namespace (fixed; not prompted) ────────────────────────────────────── + # The on-prem client is one-per-machine and is identified to the backend by + # its credentials (clientId), not by this name — so we don't ask the user to + # invent one. It's just the local k8s namespace / Helm release name. + # Advanced / GitOps setups can override with TB_NAMESPACE=. + TB_NAMESPACE=$(_sanitize_workspace_name "${TB_NAMESPACE:-tracebloc}") # ── Step 4/4: Connect to tracebloc network ────────────────────────────── step 4 4 "Connect to tracebloc network" @@ -255,6 +247,46 @@ install_client_helm() { default_client_id=""; default_client_password="" done + # ── One-client-per-machine guard ───────────────────────────────────────── + # A machine runs exactly one tracebloc client: it shares this cluster and the + # host's CPU/RAM/GPU, and the platform counts each client as separate + # capacity. If a DIFFERENT client is already installed here, a re-install + # would silently re-point the machine — so we stop and let the operator + # decide. The same clientId is a normal re-run/upgrade and passes through. + # Check ANY namespace: a fresh install lands in `tracebloc`, but an install + # from an older installer version may be in a different namespace. Enumerate + # client-chart releases and read each clientId. jq is already used elsewhere + # in the installer; if it's somehow absent, fall back to the `tracebloc` ns. + local existing_id="" existing_ns="" _gvf _rel _ns _id + _gvf="$(mktemp)" + if has jq; then + while IFS=$'\t' read -r _rel _ns; do + [[ -z "$_rel" ]] && continue + if helm get values "$_rel" -n "$_ns" > "$_gvf" 2>/dev/null; then + _id="$(_extract_yaml_value "$_gvf" clientId)" + [[ -n "$_id" ]] && { existing_id="$_id"; existing_ns="$_ns"; break; } + fi + done < <(helm list -A -o json 2>/dev/null | jq -r '.[] | select((.chart // "") | startswith("client-")) | "\(.name)\t\(.namespace)"') + elif helm get values "$TB_NAMESPACE" -n "$TB_NAMESPACE" > "$_gvf" 2>/dev/null; then + existing_id="$(_extract_yaml_value "$_gvf" clientId)"; existing_ns="$TB_NAMESPACE" + fi + rm -f "$_gvf" + if [[ -n "$existing_id" && "$existing_id" != "$TB_CLIENT_ID" ]]; then + echo "" + warn "This machine already runs the tracebloc client '${existing_id}' (namespace '${existing_ns}')." + hint "tracebloc runs one client per machine — it shares this cluster and host" + hint "resources, and the platform counts each client as separate capacity." + echo "" + hint "You entered a different Client ID ('${TB_CLIENT_ID}'). Pick one:" + hint " • Repair / update '${existing_id}' → re-run with that same Client ID" + hint " • Switch to '${TB_CLIENT_ID}' → remove the current client first:" + hint " k3d cluster delete ${CLUSTER_NAME:-tracebloc} (wipes this client + its local data)" + hint " then re-run this installer" + hint " • Run both clients → install on a separate machine" + echo "" + error "Refusing to replace the existing client. See the options above." + fi + TB_CLIENT_PASSWORD_ESCAPED="${TB_CLIENT_PASSWORD//\'/\'\'}" # ── GPU limits ────────────────────────────────────────────────────────── diff --git a/scripts/lib/summary.sh b/scripts/lib/summary.sh index 3a501bb..ccf9f93 100755 --- a/scripts/lib/summary.sh +++ b/scripts/lib/summary.sh @@ -89,6 +89,7 @@ print_summary() { [[ "$GPU_VENDOR" == "nvidia" ]] && mode="NVIDIA GPU" [[ "$GPU_VENDOR" == "amd" ]] && mode="AMD GPU" local ns="${TB_NAMESPACE:-default}" + local cver; cver="$(_chart_version "$ns")" local line="━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" echo "" @@ -99,6 +100,7 @@ print_summary() { echo -e " ${BOLD}${GREEN}✔ Connected to tracebloc${RESET}" echo "" echo -e " ${BOLD}Workspace${RESET} : ${CYAN}${ns}${RESET}" + echo -e " ${BOLD}Version${RESET} : ${CYAN}${cver:-unknown}${RESET}" echo -e " ${BOLD}Mode${RESET} : ${CYAN}${mode}${RESET}" echo "" echo -e " Your client is live. Confirm it shows as ${BOLD}🟢 Online${RESET}:" diff --git a/scripts/tests/diagnose.bats b/scripts/tests/diagnose.bats index 93f4b97..0e6f533 100644 --- a/scripts/tests/diagnose.bats +++ b/scripts/tests/diagnose.bats @@ -104,3 +104,14 @@ setup() { # Finding 2 (security review): `helm get manifest` (base64 Secrets) is NOT collected ! tar -xzOf "$tgz" 2>/dev/null | grep -q 'get manifest' } + +@test "run_diagnose: surfaces + records the client version" { + has() { case "$1" in helm) return 0 ;; *) return 1 ;; esac; } # only helm present + helm() { echo "tracebloc tracebloc 1 now deployed client-1.4.4 1.4.4"; } + run run_diagnose + [ "$status" -eq 0 ] + [[ "$output" == *"client version: 1.4.4"* ]] + tgz="$(ls "$HOST_DATA_DIR"/tracebloc-diagnose-*.tgz 2>/dev/null | head -1)" + [ -n "$tgz" ] + tar -xzOf "$tgz" 2>/dev/null | grep -q 'CLIENT VERSION: 1.4.4' +} diff --git a/scripts/tests/install-client-helm.bats b/scripts/tests/install-client-helm.bats index 739a96d..e9565bc 100644 --- a/scripts/tests/install-client-helm.bats +++ b/scripts/tests/install-client-helm.bats @@ -126,13 +126,13 @@ setup() { _ensure_helm_runnable() { :; } helm() { record "helm $*"; return 0; } verify_credentials() { printf valid; } - run install_client_helm <<< $'myws\nmyid\nmypw' + run install_client_helm <<< $'myid\nmypw' [ "$status" -eq 0 ] [[ "$output" == *"Credentials verified"* ]] [[ "$output" == *"Connected to tracebloc"* ]] grep -q 'clientId: "myid"' "$HOST_DATA_DIR/values.yaml" grep -q "clientPassword: 'mypw'" "$HOST_DATA_DIR/values.yaml" - mock_calls | grep -q "helm upgrade --install myws" + mock_calls | grep -q "helm upgrade --install tracebloc" } @test "install_client_helm: re-prompts on invalid, then accepts valid" { @@ -145,7 +145,7 @@ setup() { local n; n=$(cat "$BATS_TEST_TMPDIR/n" 2>/dev/null || echo 0); n=$((n+1)); echo "$n" >"$BATS_TEST_TMPDIR/n" if [ "$n" -ge 2 ]; then printf valid; else printf invalid; fi } - run install_client_helm <<< $'myws\nbadid\nbadpw\ngoodid\ngoodpw' + run install_client_helm <<< $'badid\nbadpw\ngoodid\ngoodpw' [ "$status" -eq 0 ] [[ "$output" == *"rejected"* ]] [[ "$output" == *"Credentials verified"* ]] @@ -159,7 +159,7 @@ setup() { _ensure_helm_runnable() { :; } helm() { record "helm $*"; return 0; } verify_credentials() { printf inactive; } - run install_client_helm <<< $'myws\nmyid\nmypw' + run install_client_helm <<< $'myid\nmypw' [ "$status" -ne 0 ] [[ "$output" == *"not active"* ]] run mock_calls @@ -173,7 +173,7 @@ setup() { _ensure_helm_runnable() { :; } helm() { record "helm $*"; return 0; } verify_credentials() { printf unverified; } - run install_client_helm <<< $'myws\nmyid\nmypw' + run install_client_helm <<< $'myid\nmypw' [ "$status" -eq 0 ] [[ "$output" == *"Couldn't reach tracebloc"* ]] run mock_calls @@ -202,8 +202,8 @@ setup() { _ensure_helm_runnable() { :; } helm() { record "helm $*"; return 0; } verify_credentials() { printf valid; } - # use-previous=y, workspace=myws, ClientID=Enter(keep previd), password=Enter(keep prevpw) - run install_client_helm <<< $'y\nmyws\n\n\n' + # use-previous=y, ClientID=Enter(keep previd), password=Enter(keep prevpw) + run install_client_helm <<< $'y\n\n\n' [ "$status" -eq 0 ] grep -q 'clientId: "previd"' "$HOST_DATA_DIR/values.yaml" grep -q "clientPassword: 'prevpw'" "$HOST_DATA_DIR/values.yaml" @@ -216,9 +216,47 @@ setup() { _ensure_helm_runnable() { :; } helm() { record "helm $*"; return 0; } verify_credentials() { printf invalid; } - run install_client_helm <<< $'myws\ni1\np1\ni2\np2\ni3\np3\ni4\np4\ni5\np5' + run install_client_helm <<< $'i1\np1\ni2\np2\ni3\np3\ni4\np4\ni5\np5' [ "$status" -ne 0 ] [[ "$output" == *"Too many failed attempts"* ]] run mock_calls [[ "$output" != *"helm upgrade"* ]] } + +# ── One-client-per-machine guard ──────────────────────────────────────────── +@test "install_client_helm: blocks a DIFFERENT client already installed" { + HOST_DATA_DIR="$BATS_TEST_TMPDIR/data"; mkdir -p "$HOST_DATA_DIR" + _ensure_tracebloc_dirs() { :; } + _ensure_release_dirs() { :; } + _ensure_helm_runnable() { :; } + # an existing release reports a different clientId -> must block before upgrade + helm() { + if [ "$1" = list ]; then echo '[{"name":"oldrel","namespace":"default","chart":"client-1.4.3"}]'; return 0; fi + if [ "$1" = get ] && [ "$2" = values ]; then echo 'clientId: "otherclient"'; return 0; fi + record "helm $*"; return 0 + } + verify_credentials() { printf valid; } + run install_client_helm <<< $'newclient\nmypw' + [ "$status" -ne 0 ] + [[ "$output" == *"already runs the tracebloc client 'otherclient'"* ]] + [[ "$output" == *"one client per machine"* ]] + run mock_calls + [[ "$output" != *"helm upgrade"* ]] +} + +@test "install_client_helm: same client re-run is allowed (upgrade in place)" { + HOST_DATA_DIR="$BATS_TEST_TMPDIR/data"; mkdir -p "$HOST_DATA_DIR" + _ensure_tracebloc_dirs() { :; } + _ensure_release_dirs() { :; } + _ensure_helm_runnable() { :; } + helm() { + if [ "$1" = list ]; then echo '[{"name":"tracebloc","namespace":"tracebloc","chart":"client-1.4.3"}]'; return 0; fi + if [ "$1" = get ] && [ "$2" = values ]; then echo 'clientId: "sameid"'; return 0; fi + record "helm $*"; return 0 + } + verify_credentials() { printf valid; } + run install_client_helm <<< $'sameid\nmypw' + [ "$status" -eq 0 ] + run mock_calls + [[ "$output" == *"helm upgrade --install tracebloc"* ]] +} diff --git a/scripts/tests/install-k8s.Tests.ps1 b/scripts/tests/install-k8s.Tests.ps1 index ed3967c..fe39098 100644 --- a/scripts/tests/install-k8s.Tests.ps1 +++ b/scripts/tests/install-k8s.Tests.ps1 @@ -98,6 +98,13 @@ Describe "Print-Summary" { $out = Print-Summary 6>&1 | Out-String $out | Should -Match "crash loop" } + It "connected: shows the client version" { + $script:ClientState = "connected" + Mock helm { "tracebloc tracebloc 1 now deployed client-1.4.4 1.4.4" } + $out = Print-Summary 6>&1 | Out-String + $out | Should -Match "Version" + $out | Should -Match "1\.4\.4" + } } Describe "ConvertTo-WorkspaceName" { @@ -228,6 +235,39 @@ Describe "Install-ClientHelm" { Install-ClientHelm (Get-Content "$HOST_DATA_DIR/values.yaml" -Raw) | Should -Match 'clientId: "previd"' } + It "blocks a DIFFERENT client already installed" { + $HOST_DATA_DIR = "$TestDrive/d5" + Mock Err { throw "err" } + Mock Read-Host { + param([string]$Prompt, [switch]$AsSecureString) + if ($Prompt -match 'password') { return (ConvertTo-SecureString "pw" -AsPlainText -Force) } + return "newclient" + } + Mock Test-Credentials { "valid" } + Mock helm { + if ($args -contains "list") { '[{"name":"oldrel","namespace":"default","chart":"client-1.4.3"}]'; $global:LASTEXITCODE = 0; return } + if ($args -contains "get") { 'clientId: "otherclient"'; $global:LASTEXITCODE = 0; return } + $global:LASTEXITCODE = 0 + } + { Install-ClientHelm } | Should -Throw + Should -Not -Invoke helm -ParameterFilter { $args -contains "upgrade" } + } + It "same client re-run is allowed (upgrade in place)" { + $HOST_DATA_DIR = "$TestDrive/d6" + Mock Read-Host { + param([string]$Prompt, [switch]$AsSecureString) + if ($Prompt -match 'password') { return (ConvertTo-SecureString "pw" -AsPlainText -Force) } + return "sameid" + } + Mock Test-Credentials { "valid" } + Mock helm { + if ($args -contains "list") { '[{"name":"tracebloc","namespace":"tracebloc","chart":"client-1.4.3"}]'; $global:LASTEXITCODE = 0; return } + if ($args -contains "get") { 'clientId: "sameid"'; $global:LASTEXITCODE = 0; return } + $global:LASTEXITCODE = 0 + } + Install-ClientHelm + Should -Invoke helm -ParameterFilter { $args -contains "upgrade" } + } } Describe "Confirm-Cluster" { diff --git a/scripts/tests/summary.bats b/scripts/tests/summary.bats index 924080a..7f0b91f 100644 --- a/scripts/tests/summary.bats +++ b/scripts/tests/summary.bats @@ -64,6 +64,14 @@ setup() { [[ "$output" == *"data never leaves"* ]] } +@test "print_summary connected: shows the client version" { + CLIENT_STATE=connected + helm() { echo "tracebloc tracebloc 1 now deployed client-1.4.4 1.4.4"; } + run print_summary + [[ "$output" == *"Version"* ]] + [[ "$output" == *"1.4.4"* ]] +} + @test "print_summary starting: 'still starting', no trust claim" { CLIENT_STATE=starting run print_summary From ddb233a48b3af1b266724c60d9222c9819494d49 Mon Sep 17 00:00:00 2001 From: "Asad Iqbal (Saadi)" Date: Thu, 4 Jun 2026 14:38:12 +0500 Subject: [PATCH 9/9] =?UTF-8?q?chore:=20bump=20chart=201.4.4=20=E2=86=92?= =?UTF-8?q?=201.4.5=20to=20ship=20the=20training-egress=20proxy=20fix=20(#?= =?UTF-8?q?198)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 1.4.4 is already published on the tracebloc.github.io/client Pages channel and is what clusters run. The training-egress NetworkPolicy fix (#197, allow training → requests-proxy:8888) merged to develop without a version bump, so it is currently undeliverable: chart-releaser won't overwrite the existing 1.4.4 release, and clusters already on 1.4.4 would see no version change and pull nothing. Bump to 1.4.5 (lockstep version/appVersion, matching 1.4.3/1.4.4 history) so a v1.4.5 release publishes a new version that auto-upgrade actually pulls. Chart-only change; no image change. Ref #196 / #197. Co-authored-by: Claude Opus 4.8 (1M context) --- client/Chart.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/client/Chart.yaml b/client/Chart.yaml index 0e495fc..052c031 100644 --- a/client/Chart.yaml +++ b/client/Chart.yaml @@ -2,8 +2,8 @@ apiVersion: v2 name: client description: A unified Helm chart for tracebloc on AKS, EKS, bare-metal, and OpenShift type: application -version: 1.4.4 -appVersion: "1.4.4" +version: 1.4.5 +appVersion: "1.4.5" keywords: - tracebloc - kubernetes