From 1a17c0044ea690987cfb4e823062a0b2db8c56f5 Mon Sep 17 00:00:00 2001
From: Jordan Olshevski <jordan.olshevski@microsoft.com>
Date: Wed, 28 Oct 2026 17:26:23 +0000
Subject: [PATCH 01/28] hack: add gantry vs gantry+racer pull benchmark harness

---
 hack/gantry-racer-bench/bench.sh         | 290 ++++++++++++++++++++
 hack/gantry-racer-bench/genimage/main.go | 327 +++++++++++++++++++++++
 hack/gantry-racer-bench/origin.yaml      | 101 +++++++
 3 files changed, 718 insertions(+)
 create mode 100755 hack/gantry-racer-bench/bench.sh
 create mode 100644 hack/gantry-racer-bench/genimage/main.go
 create mode 100644 hack/gantry-racer-bench/origin.yaml

diff --git a/hack/gantry-racer-bench/bench.sh b/hack/gantry-racer-bench/bench.sh
new file mode 100755
index 000000000..65c60cdb1
--- /dev/null
+++ b/hack/gantry-racer-bench/bench.sh
@@ -0,0 +1,290 @@
+#!/usr/bin/env bash
+# Gantry vs Gantry+Racer pull benchmark harness.
+#
+# Subcommands:
+#   prep                 install containerd hosts.toml for the bench registry on every worker
+#   gen <name> <layers> <layer-bytes>
+#                        push a fresh random image to the origin, print repo@digest
+#   run <label> <repo@digest> <mode> [concurrency]
+#                        start every worker pulling at the same instant and collect results
+#                        mode=crictl: full containerd pull + unpack
+#                        mode=curl:   fetch every layer blob through the local gantry mirror
+#   summarize <results.tsv>
+set -euo pipefail
+
+NS=gantry-bench
+HOST=bench.registry.local
+ORIGIN=${ORIGIN:-10.0.90.91:5000}
+DIR=$(cd "$(dirname "$0")" && pwd)
+OUT=${OUT:-$DIR/../../tmp/gantry-racer-bench}
+LEAD=${LEAD:-240}
+BUSYBOX=mcr.microsoft.com/azurelinux/busybox:1.36
+
+mkdir -p "$OUT"
+
+tools_pod() {
+	kubectl -n "$NS" get pod -l app=tools -o jsonpath='{.items[0].metadata.name}'
+}
+
+worker_affinity() {
+	cat <<'EOF'
+      affinity:
+        nodeAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            nodeSelectorTerms:
+              - matchExpressions:
+                  - key: kubernetes.azure.com/mode
+                    operator: In
+                    values: ["user"]
+EOF
+}
+
+cmd_prep() {
+	kubectl apply -f - <<EOF
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: prep
+  namespace: $NS
+spec:
+  updateStrategy:
+    rollingUpdate:
+      maxUnavailable: 100%
+  selector:
+    matchLabels:
+      app: prep
+  template:
+    metadata:
+      labels:
+        app: prep
+    spec:
+      terminationGracePeriodSeconds: 1
+$(worker_affinity)
+      tolerations:
+        - operator: Exists
+      containers:
+        - name: prep
+          image: $BUSYBOX
+          securityContext:
+            privileged: true
+          command:
+            - chroot
+            - /host
+            - bash
+            - -c
+            - |
+              set -e
+              mkdir -p /etc/containerd/certs.d/$HOST
+              cat > /etc/containerd/certs.d/$HOST/hosts.toml <<'TOML'
+              # Fall back to the origin itself, as a real registry host would.
+              server = "http://$ORIGIN"
+              [host."http://127.0.0.1:5000"]
+                capabilities = ["pull", "resolve"]
+              TOML
+              echo done
+              exec sleep infinity
+          resources:
+            requests:
+              cpu: 10m
+              memory: 16Mi
+          volumeMounts:
+            - name: host
+              mountPath: /host
+      volumes:
+        - name: host
+          hostPath:
+            path: /
+EOF
+	kubectl -n "$NS" rollout status ds/prep --timeout=15m
+}
+
+cmd_gen() {
+	local name=$1 layers=$2 bytes=$3
+	kubectl -n "$NS" exec "$(tools_pod)" -- /tmp/genimage -registry http://origin:5000 \
+		-repo "bench/$name" -layers "$layers" -layer-bytes "$bytes" -parallel 8
+}
+
+cmd_run() {
+	local label=$1 ref=$2 mode=$3 conc=${4:-3}
+	local repo=${ref%@*} digest=${ref#*@}
+	local start_at=$(($(date +%s) + LEAD))
+	local name="pull-$(date +%s)"
+	local layers=""
+
+	if [ "$mode" = curl ]; then
+		layers=$(kubectl -n "$NS" exec "$(tools_pod)" -- wget -qO- \
+			--header 'Accept: application/vnd.oci.image.manifest.v1+json' \
+			"http://origin:5000/v2/$repo/manifests/$digest" |
+			python3 -c 'import json,sys; print(" ".join(l["digest"] for l in json.load(sys.stdin)["layers"]))')
+	fi
+
+	echo "run=$label ds=$name start_at=$start_at ($(date -d @"$start_at" -u +%T)) ref=$ref mode=$mode" >&2
+
+	kubectl apply -f - <<EOF
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: $name
+  namespace: $NS
+  labels:
+    app: pull
+spec:
+  selector:
+    matchLabels:
+      app: pull
+      run: $name
+  template:
+    metadata:
+      labels:
+        app: pull
+        run: $name
+    spec:
+$(worker_affinity)
+      tolerations:
+        - operator: Exists
+      terminationGracePeriodSeconds: 1
+      containers:
+        - name: pull
+          image: $BUSYBOX
+          securityContext:
+            privileged: true
+          env:
+            - name: NODE
+              valueFrom:
+                fieldRef:
+                  fieldPath: spec.nodeName
+            - name: START_AT
+              value: "$start_at"
+            - name: REF
+              value: "$HOST/$repo@$digest"
+            - name: REPO
+              value: "$repo"
+            - name: MODE
+              value: "$mode"
+            - name: LAYERS
+              value: "$layers"
+            - name: CONC
+              value: "$conc"
+          command:
+            - chroot
+            - /host
+            - bash
+            - -c
+            - |
+              set -u
+              CRI="crictl --runtime-endpoint unix:///run/containerd/containerd.sock --image-endpoint unix:///run/containerd/containerd.sock --timeout 2h"
+              # Drop images from earlier rounds so the disk does not fill.
+              for id in \$(\$CRI images 2>/dev/null | awk '\$1 ~ /^$HOST\// {print \$3}' | sort -u); do
+                \$CRI rmi "\$id" >/dev/null 2>&1 || true
+              done
+              echo "READY \$NODE \$(date +%s.%N)"
+              while [ "\$(date +%s)" -lt "\$START_AT" ]; do sleep 0.02; done
+              t0=\$(date +%s.%N)
+              rc=0
+              bytes=0
+              if [ "\$MODE" = crictl ]; then
+                out=\$(\$CRI pull "\$REF" 2>&1) || rc=\$?
+              else
+                tmp=\$(mktemp)
+                printf '%s\n' \$LAYERS | xargs -P "\$CONC" -I{} curl -sS -o /dev/null -w '%{http_code} %{size_download}\n' \
+                  "http://127.0.0.1:5000/v2/\$REPO/blobs/{}?ns=$HOST" > "\$tmp" 2>&1 || rc=\$?
+                bytes=\$(awk '{s+=\$2} END {print s+0}' "\$tmp")
+                bad=\$(awk '\$1 != 200' "\$tmp" | wc -l)
+                [ "\$bad" -eq 0 ] || rc=99
+                out=\$(head -c 300 "\$tmp" | tr '\n' ' ')
+                rm -f "\$tmp"
+              fi
+              t1=\$(date +%s.%N)
+              echo "RESULT \$NODE \$MODE \$t0 \$t1 \$rc \$bytes"
+              [ "\$rc" -eq 0 ] || echo "ERROR \$NODE \$(echo "\$out" | tr '\n' ' ' | head -c 500)"
+              exec sleep infinity
+          resources:
+            requests:
+              cpu: 10m
+              memory: 16Mi
+          volumeMounts:
+            - name: host
+              mountPath: /host
+      volumes:
+        - name: host
+          hostPath:
+            path: /
+EOF
+
+	local results="$OUT/$label.tsv"
+	local want
+	want=$(kubectl get nodes -l kubernetes.azure.com/mode=user --no-headers | wc -l)
+
+	# Wait for every pod to report a result.
+	while true; do
+		sleep 20
+		collect "$name" >"$results.tmp"
+		local got
+		got=$(grep -c '^RESULT' "$results.tmp" || true)
+		echo "$(date -u +%T) results $got/$want" >&2
+		if [ "$got" -ge "$want" ]; then
+			break
+		fi
+		if [ "$(date +%s)" -gt $((start_at + ${MAX_WAIT:-3600})) ]; then
+			echo "timed out waiting for results" >&2
+			break
+		fi
+	done
+
+	mv "$results.tmp" "$results"
+	late=$(awk -v s="$start_at" '$1 == "READY" && $3 > s' "$results" | wc -l)
+	echo "pods ready after start: $late" >&2
+	kubectl -n "$NS" delete ds "$name" --wait=false >/dev/null
+	cmd_summarize "$results" "$start_at"
+}
+
+# collect prints READY/RESULT/ERROR lines from every pod of a run.
+collect() {
+	local name=$1
+	kubectl -n "$NS" get pods -l run="$name" -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}' |
+		xargs -P 64 -I{} sh -c "kubectl -n $NS logs {} 2>/dev/null | grep -E '^(READY|RESULT|ERROR)' || true"
+}
+
+cmd_summarize() {
+	local results=$1 start_at=${2:-}
+	python3 - "$results" "$start_at" <<'EOF'
+import sys, statistics
+path, start_at = sys.argv[1], sys.argv[2]
+rows, errors = [], []
+for line in open(path):
+    f = line.split()
+    if f[0] == "RESULT":
+        rows.append((f[1], float(f[3]), float(f[4]), int(f[5]), int(f[6])))
+    elif f[0] == "ERROR":
+        errors.append(line.strip())
+if not rows:
+    print("no results"); sys.exit()
+t0 = float(start_at) if start_at else min(r[1] for r in rows)
+def pct(xs, p):
+    xs = sorted(xs)
+    return xs[min(len(xs) - 1, int(round(p / 100 * (len(xs) - 1))))]
+def line(name, rs):
+    d = [r[2] - t0 for r in rs if r[3] == 0]
+    fails = sum(1 for r in rs if r[3] != 0)
+    if not d:
+        print(f"{name:10s} n={len(rs)} failed={fails}"); return
+    print(f"{name:10s} n={len(rs):5d} ok={len(d):5d} failed={fails:4d} "
+          f"p50={pct(d,50):7.1f}s p90={pct(d,90):7.1f}s p95={pct(d,95):7.1f}s p99={pct(d,99):7.1f}s "
+          f"max={max(d):7.1f}s mean={statistics.mean(d):7.1f}s")
+line("all", rows)
+for pool in sorted({r[0].split("-")[1] for r in rows}):
+    line(pool, [r for r in rows if r[0].split("-")[1] == pool])
+skew = [r[1] - t0 for r in rows]
+print(f"start skew: max {max(skew):.2f}s")
+for e in errors[:5]:
+    print(e)
+EOF
+}
+
+case ${1:-} in
+prep) cmd_prep ;;
+gen) shift; cmd_gen "$@" ;;
+run) shift; cmd_run "$@" ;;
+summarize) shift; cmd_summarize "$@" ;;
+*) sed -n '2,13p' "$0"; exit 1 ;;
+esac
diff --git a/hack/gantry-racer-bench/genimage/main.go b/hack/gantry-racer-bench/genimage/main.go
new file mode 100644
index 000000000..f2d2f7fae
--- /dev/null
+++ b/hack/gantry-racer-bench/genimage/main.go
@@ -0,0 +1,327 @@
+// Command genimage pushes an OCI image made of random, incompressible layers
+// to a plain HTTP registry. Every run uses a different seed so each benchmark
+// round starts from content that no cache has seen.
+//
+// It depends only on the standard library so it can be cross-compiled and
+// copied into any pod next to the origin registry.
+package main
+
+import (
+	"archive/tar"
+	"bytes"
+	"compress/gzip"
+	"crypto/sha256"
+	"encoding/hex"
+	"encoding/json"
+	"flag"
+	"fmt"
+	"hash"
+	"io"
+	"log"
+	"math/rand/v2"
+	"net/http"
+	"net/url"
+	"os"
+	"strings"
+	"sync"
+	"time"
+)
+
+type descriptor struct {
+	MediaType string `json:"mediaType"`
+	Digest    string `json:"digest"`
+	Size      int64  `json:"size"`
+}
+
+type layerResult struct {
+	desc   descriptor
+	diffID string
+}
+
+func main() {
+	registry := flag.String("registry", "http://127.0.0.1:5000", "registry base URL")
+	repo := flag.String("repo", "bench/img", "repository name")
+	tag := flag.String("tag", "latest", "tag to push")
+	layers := flag.Int("layers", 8, "number of layers")
+	layerBytes := flag.Int64("layer-bytes", 1<<30, "random payload bytes per layer")
+	seed := flag.Uint64("seed", uint64(time.Now().UnixNano()), "PRNG seed")
+	parallel := flag.Int("parallel", 8, "layers generated and pushed in parallel")
+
+	flag.Parse()
+
+	start := time.Now()
+	results := make([]layerResult, *layers)
+	errs := make([]error, *layers)
+	sem := make(chan struct{}, *parallel)
+
+	var wg sync.WaitGroup
+
+	for i := range *layers {
+		wg.Add(1)
+
+		go func() {
+			defer wg.Done()
+
+			sem <- struct{}{}
+			defer func() { <-sem }()
+
+			results[i], errs[i] = pushLayer(*registry, *repo, i, *layerBytes, *seed)
+		}()
+	}
+
+	wg.Wait()
+
+	for _, err := range errs {
+		if err != nil {
+			log.Fatal(err)
+		}
+	}
+
+	diffIDs := make([]string, 0, len(results))
+	layerDescs := make([]descriptor, 0, len(results))
+
+	for _, r := range results {
+		diffIDs = append(diffIDs, r.diffID)
+		layerDescs = append(layerDescs, r.desc)
+	}
+
+	config := map[string]any{
+		"architecture": "amd64",
+		"os":           "linux",
+		"created":      time.Now().UTC().Format(time.RFC3339),
+		"config":       map[string]any{"Cmd": []string{"/data/none"}},
+		"rootfs":       map[string]any{"type": "layers", "diff_ids": diffIDs},
+	}
+
+	configBytes, err := json.Marshal(config)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	configDigest, err := pushBlob(*registry, *repo, bytes.NewReader(configBytes), nil)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	manifest := map[string]any{
+		"schemaVersion": 2,
+		"mediaType":     "application/vnd.oci.image.manifest.v1+json",
+		"config": descriptor{
+			MediaType: "application/vnd.oci.image.config.v1+json",
+			Digest:    configDigest,
+			Size:      int64(len(configBytes)),
+		},
+		"layers": layerDescs,
+	}
+
+	manifestBytes, err := json.Marshal(manifest)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	sum := sha256.Sum256(manifestBytes)
+	manifestDigest := "sha256:" + hex.EncodeToString(sum[:])
+
+	req, err := http.NewRequest(http.MethodPut, fmt.Sprintf("%s/v2/%s/manifests/%s", *registry, *repo, *tag), bytes.NewReader(manifestBytes))
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	req.Header.Set("Content-Type", "application/vnd.oci.image.manifest.v1+json")
+
+	if err := expect(http.DefaultClient.Do(req)); err != nil {
+		log.Fatalf("put manifest: %v", err)
+	}
+
+	var total int64
+	for _, d := range layerDescs {
+		total += d.Size
+	}
+
+	fmt.Fprintf(os.Stderr, "pushed %d layers, %d bytes in %s\n", len(layerDescs), total, time.Since(start).Round(time.Millisecond))
+	fmt.Printf("%s@%s\n", *repo, manifestDigest)
+}
+
+// pushLayer streams one gzip(tar(random file)) layer to the registry and
+// returns its descriptor and uncompressed diff ID. Random data does not
+// compress, so gzip at NoCompression produces the same stored-block stream
+// a real compressor would and decompression cost stays realistic.
+func pushLayer(registry, repo string, index int, size int64, seed uint64) (layerResult, error) {
+	pr, pw := io.Pipe()
+	diff := sha256.New()
+
+	go func() {
+		pw.CloseWithError(writeLayer(pw, diff, index, size, seed))
+	}()
+
+	counter := &countingReader{r: pr}
+
+	digest, err := pushBlob(registry, repo, counter, nil)
+	if err != nil {
+		return layerResult{}, fmt.Errorf("layer %d: %w", index, err)
+	}
+
+	return layerResult{
+		desc: descriptor{
+			MediaType: "application/vnd.oci.image.layer.v1.tar+gzip",
+			Digest:    digest,
+			Size:      counter.n,
+		},
+		diffID: "sha256:" + hex.EncodeToString(diff.Sum(nil)),
+	}, nil
+}
+
+func writeLayer(w io.Writer, diff hash.Hash, index int, size int64, seed uint64) error {
+	gz, err := gzip.NewWriterLevel(w, gzip.NoCompression)
+	if err != nil {
+		return err
+	}
+
+	tw := tar.NewWriter(io.MultiWriter(gz, diff))
+
+	hdr := &tar.Header{
+		Name:     fmt.Sprintf("data/layer-%02d.bin", index),
+		Mode:     0o644,
+		Size:     size,
+		ModTime:  time.Unix(0, 0),
+		Typeflag: tar.TypeReg,
+	}
+
+	if err := tw.WriteHeader(hdr); err != nil {
+		return err
+	}
+
+	var key [32]byte
+	for i := range key {
+		key[i] = byte(seed >> (8 * (i % 8)))
+	}
+
+	key[31] = byte(index)
+	key[30] = byte(index >> 8)
+
+	rng := rand.NewChaCha8(key)
+	buf := make([]byte, 1<<20)
+
+	for remaining := size; remaining > 0; {
+		n := int64(len(buf))
+		if remaining < n {
+			n = remaining
+		}
+
+		_, _ = rng.Read(buf[:n])
+
+		if _, err := tw.Write(buf[:n]); err != nil {
+			return err
+		}
+
+		remaining -= n
+	}
+
+	if err := tw.Close(); err != nil {
+		return err
+	}
+
+	return gz.Close()
+}
+
+// pushBlob uploads body with a streaming PATCH and commits it by digest.
+func pushBlob(registry, repo string, body io.Reader, _ http.Header) (string, error) {
+	resp, err := http.Post(fmt.Sprintf("%s/v2/%s/blobs/uploads/", registry, repo), "application/octet-stream", nil)
+	if err != nil {
+		return "", err
+	}
+
+	_ = resp.Body.Close()
+
+	if resp.StatusCode != http.StatusAccepted {
+		return "", fmt.Errorf("start upload: %s", resp.Status)
+	}
+
+	location, err := resolve(registry, resp.Header.Get("Location"))
+	if err != nil {
+		return "", err
+	}
+
+	h := sha256.New()
+
+	req, err := http.NewRequest(http.MethodPatch, location, io.TeeReader(body, h))
+	if err != nil {
+		return "", err
+	}
+
+	req.Header.Set("Content-Type", "application/octet-stream")
+
+	resp, err = http.DefaultClient.Do(req)
+	if err != nil {
+		return "", err
+	}
+
+	_ = resp.Body.Close()
+
+	if resp.StatusCode != http.StatusAccepted {
+		return "", fmt.Errorf("patch upload: %s", resp.Status)
+	}
+
+	location, err = resolve(registry, resp.Header.Get("Location"))
+	if err != nil {
+		return "", err
+	}
+
+	digest := "sha256:" + hex.EncodeToString(h.Sum(nil))
+
+	sep := "?"
+	if strings.Contains(location, "?") {
+		sep = "&"
+	}
+
+	req, err = http.NewRequest(http.MethodPut, location+sep+"digest="+url.QueryEscape(digest), nil)
+	if err != nil {
+		return "", err
+	}
+
+	if err := expect(http.DefaultClient.Do(req)); err != nil {
+		return "", fmt.Errorf("commit upload: %w", err)
+	}
+
+	return digest, nil
+}
+
+func resolve(base, location string) (string, error) {
+	b, err := url.Parse(base)
+	if err != nil {
+		return "", err
+	}
+
+	l, err := url.Parse(location)
+	if err != nil {
+		return "", err
+	}
+
+	return b.ResolveReference(l).String(), nil
+}
+
+func expect(resp *http.Response, err error) error {
+	if err != nil {
+		return err
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode/100 != 2 {
+		msg, _ := io.ReadAll(io.LimitReader(resp.Body, 4096))
+		return fmt.Errorf("%s: %s", resp.Status, msg)
+	}
+
+	return nil
+}
+
+type countingReader struct {
+	r io.Reader
+	n int64
+}
+
+func (c *countingReader) Read(p []byte) (int, error) {
+	n, err := c.r.Read(p)
+	c.n += int64(n)
+
+	return n, err
+}
diff --git a/hack/gantry-racer-bench/origin.yaml b/hack/gantry-racer-bench/origin.yaml
new file mode 100644
index 000000000..473941ee7
--- /dev/null
+++ b/hack/gantry-racer-bench/origin.yaml
@@ -0,0 +1,101 @@
+# In-cluster origin registry for the Gantry vs Gantry+Racer benchmark.
+# Plain HTTP distribution registry pinned to one system node, backed by the
+# node's local disk, so origin bandwidth is the same for every run.
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: gantry-bench
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: origin
+  namespace: gantry-bench
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: origin
+  template:
+    metadata:
+      labels:
+        app: origin
+    spec:
+      nodeSelector:
+        kubernetes.io/hostname: aks-system-36530849-vmss000002
+      containers:
+        - name: registry
+          image: ghcr.io/distribution/distribution:3.0.0
+          env:
+            - name: REGISTRY_HTTP_ADDR
+              value: ":5000"
+            - name: REGISTRY_STORAGE_DELETE_ENABLED
+              value: "true"
+            - name: REGISTRY_LOG_LEVEL
+              value: warn
+            - name: OTEL_TRACES_EXPORTER
+              value: none
+            - name: REGISTRY_HTTP_DEBUG_ADDR
+              value: ":5001"
+            - name: REGISTRY_HTTP_DEBUG_PROMETHEUS_ENABLED
+              value: "true"
+          ports:
+            - containerPort: 5000
+              name: http
+            - containerPort: 5001
+              name: debug
+          resources:
+            requests:
+              cpu: "4"
+              memory: 4Gi
+          volumeMounts:
+            - name: data
+              mountPath: /var/lib/registry
+      volumes:
+        - name: data
+          hostPath:
+            path: /mnt/gantry-bench-origin
+            type: DirectoryOrCreate
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: origin
+  namespace: gantry-bench
+spec:
+  selector:
+    app: origin
+  ports:
+    - name: http
+      port: 5000
+      targetPort: 5000
+    - name: debug
+      port: 5001
+      targetPort: 5001
+---
+# Tool pod next to the origin: runs genimage and scrapes fleet metrics.
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: tools
+  namespace: gantry-bench
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: tools
+  template:
+    metadata:
+      labels:
+        app: tools
+    spec:
+      nodeSelector:
+        kubernetes.io/hostname: aks-system-36530849-vmss000002
+      containers:
+        - name: tools
+          image: mcr.microsoft.com/azurelinux/busybox:1.36
+          command: ["sleep", "infinity"]
+          resources:
+            requests:
+              cpu: "4"
+              memory: 2Gi

From 1d5eeb4ab8760a7f76959a652d309340852a5d82 Mon Sep 17 00:00:00 2001
From: Jordan Olshevski <jordan.olshevski@microsoft.com>
Date: Wed, 28 Oct 2026 17:49:46 +0000
Subject: [PATCH 02/28] hack: gantry-racer-bench: resource disk prep and origin
 fallback

---
 hack/gantry-racer-bench/bench.sh    | 69 +++++++++++++++++++++++++++++
 hack/gantry-racer-bench/diskprep.sh | 53 ++++++++++++++++++++++
 2 files changed, 122 insertions(+)
 create mode 100644 hack/gantry-racer-bench/diskprep.sh

diff --git a/hack/gantry-racer-bench/bench.sh b/hack/gantry-racer-bench/bench.sh
index 65c60cdb1..3a165379d 100755
--- a/hack/gantry-racer-bench/bench.sh
+++ b/hack/gantry-racer-bench/bench.sh
@@ -22,6 +22,11 @@ BUSYBOX=mcr.microsoft.com/azurelinux/busybox:1.36
 
 mkdir -p "$OUT"
 
+origin_tx() {
+	kubectl -n "$NS" exec "$(tools_pod)" -- wget -qO- http://origin:5001/metrics |
+		awk '$1 == "process_network_transmit_bytes_total" {printf "%.0f\n", $2}'
+}
+
 tools_pod() {
 	kubectl -n "$NS" get pod -l app=tools -o jsonpath='{.items[0].metadata.name}'
 }
@@ -98,6 +103,61 @@ EOF
 	kubectl -n "$NS" rollout status ds/prep --timeout=15m
 }
 
+# cmd_diskprep frees the Azure resource disk (mounted at /mnt) on every worker
+# that has one, zeroes its head and tail so racer's device guard accepts it,
+# and annotates the Node so racer uses it as a raw block device.
+cmd_diskprep() {
+	kubectl -n "$NS" delete ds diskprep --ignore-not-found --wait=true >/dev/null
+	kubectl apply -f - <<EOF
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: diskprep
+  namespace: $NS
+spec:
+  selector:
+    matchLabels:
+      app: diskprep
+  template:
+    metadata:
+      labels:
+        app: diskprep
+    spec:
+$(worker_affinity)
+      hostPID: true
+      terminationGracePeriodSeconds: 1
+      tolerations:
+        - operator: Exists
+      containers:
+        - name: diskprep
+          image: $BUSYBOX
+          securityContext:
+            privileged: true
+          command: ["chroot", "/host", "nsenter", "-t", "1", "-m", "-u", "-i", "-n", "--", "bash", "-c"]
+          args:
+            - |
+$(sed 's/^/              /' "$DIR/diskprep.sh")
+          resources:
+            requests:
+              cpu: 10m
+              memory: 16Mi
+          volumeMounts:
+            - name: host
+              mountPath: /host
+      volumes:
+        - name: host
+          hostPath:
+            path: /
+EOF
+	kubectl -n "$NS" rollout status ds/diskprep --timeout=15m
+	kubectl -n "$NS" get pods -l app=diskprep -o name |
+		xargs -P 64 -I{} sh -c "kubectl -n $NS logs {} | grep '^DISK' || true" >"$OUT/disks.txt"
+	awk '$3 != "none" {print $2, $3}' "$OUT/disks.txt" |
+		xargs -P 64 -n 2 sh -c 'kubectl annotate node "$0" --overwrite "racer.unbounded-cloud.io/block-devices=^$1\$" >/dev/null'
+	awk '{print $3 == "none" ? $4 : "device"}' "$OUT/disks.txt" | sort | uniq -c
+	kubectl -n "$NS" delete ds diskprep --wait=false >/dev/null
+}
+
 cmd_gen() {
 	local name=$1 layers=$2 bytes=$3
 	kubectl -n "$NS" exec "$(tools_pod)" -- /tmp/genimage -registry http://origin:5000 \
@@ -118,6 +178,9 @@ cmd_run() {
 			python3 -c 'import json,sys; print(" ".join(l["digest"] for l in json.load(sys.stdin)["layers"]))')
 	fi
 
+	local tx0
+	tx0=$(origin_tx)
+
 	echo "run=$label ds=$name start_at=$start_at ($(date -d @"$start_at" -u +%T)) ref=$ref mode=$mode" >&2
 
 	kubectl apply -f - <<EOF
@@ -232,6 +295,9 @@ EOF
 	done
 
 	mv "$results.tmp" "$results"
+	local tx1
+	tx1=$(origin_tx)
+	echo "ORIGIN_TX $((tx1 - tx0))" >>"$results"
 	late=$(awk -v s="$start_at" '$1 == "READY" && $3 > s' "$results" | wc -l)
 	echo "pods ready after start: $late" >&2
 	kubectl -n "$NS" delete ds "$name" --wait=false >/dev/null
@@ -255,6 +321,8 @@ for line in open(path):
     f = line.split()
     if f[0] == "RESULT":
         rows.append((f[1], float(f[3]), float(f[4]), int(f[5]), int(f[6])))
+    elif f[0] == "ORIGIN_TX":
+        print(f"origin egress: {int(f[1]) / 2**30:.2f} GiB")
     elif f[0] == "ERROR":
         errors.append(line.strip())
 if not rows:
@@ -283,6 +351,7 @@ EOF
 
 case ${1:-} in
 prep) cmd_prep ;;
+diskprep) cmd_diskprep ;;
 gen) shift; cmd_gen "$@" ;;
 run) shift; cmd_run "$@" ;;
 summarize) shift; cmd_summarize "$@" ;;
diff --git a/hack/gantry-racer-bench/diskprep.sh b/hack/gantry-racer-bench/diskprep.sh
new file mode 100644
index 000000000..acf0dd2ef
--- /dev/null
+++ b/hack/gantry-racer-bench/diskprep.sh
@@ -0,0 +1,53 @@
+#!/usr/bin/env bash
+# Runs in the host namespaces on one worker (see bench.sh diskprep). Frees the
+# Azure resource disk for racer and prints "DISK <node> <by-id name> <dev>",
+# or "DISK <node> none <reason>" when the node has no usable disk.
+set -u
+node=$(hostname | tr "[:upper:]" "[:lower:]")
+rootdisk=$(lsblk -no pkname "$(findmnt -n -o SOURCE /)" | head -1)
+disk=""
+src=$(findmnt -n -o SOURCE /mnt 2>/dev/null || true)
+if [ -n "$src" ]; then
+	disk=/dev/$(lsblk -no pkname "$src" | head -1)
+	if [ "$disk" = "/dev/$rootdisk" ] || [ "$disk" = /dev/ ]; then
+		echo "DISK $node none mnt-on-root-disk"
+		exec sleep infinity
+	fi
+	if ! umount /mnt; then
+		echo "DISK $node none umount-failed"
+		exec sleep infinity
+	fi
+	# Keep the disk free across reboots.
+	sed -i 's|^\([^#].*[[:space:]]/mnt[[:space:]]\)|#\1|' /etc/fstab
+	wipefs -a "$disk" >/dev/null
+	mb=$(($(blockdev --getsize64 "$disk") / 1048576))
+	dd if=/dev/zero of="$disk" bs=1M count=2 oflag=direct status=none
+	dd if=/dev/zero of="$disk" bs=1M seek=$((mb - 2)) count=2 oflag=direct status=none
+	blockdev --rereadpt "$disk" 2>/dev/null || true
+	udevadm settle 2>/dev/null || true
+fi
+if [ -z "$disk" ]; then
+	# Already freed on an earlier pass (do not wipe again; racer may own it):
+	# report the unmounted sd disk that is not the root disk.
+	for d in /sys/block/sd*; do
+		[ -e "$d" ] || continue
+		b=$(basename "$d")
+		[ "$b" = "$rootdisk" ] && continue
+		findmnt -n -S "/dev/$b" >/dev/null && continue
+		disk=/dev/$b
+	done
+fi
+if [ -z "$disk" ]; then
+	echo "DISK $node none no-disk"
+	exec sleep infinity
+fi
+id=""
+for l in /dev/disk/by-id/wwn-* /dev/disk/by-id/scsi-3* /dev/disk/by-id/*; do
+	[ -e "$l" ] || continue
+	if [ "$(readlink -f "$l")" = "$disk" ]; then
+		id=$(basename "$l")
+		break
+	fi
+done
+echo "DISK $node ${id:-none} $disk"
+exec sleep infinity

From 72eb9ece6ee3989387c85fcc8ac36ca82c6c6ec3 Mon Sep 17 00:00:00 2001
From: Jordan Olshevski <jordan.olshevski@microsoft.com>
Date: Wed, 28 Oct 2026 18:16:34 +0000
Subject: [PATCH 03/28] hack/gantry-racer-bench: hostNetwork pull pods,
 stricter disk prep

---
 hack/gantry-racer-bench/bench.sh    |  1 +
 hack/gantry-racer-bench/diskprep.sh | 16 ++++++++++++++++
 2 files changed, 17 insertions(+)

diff --git a/hack/gantry-racer-bench/bench.sh b/hack/gantry-racer-bench/bench.sh
index 3a165379d..8d50368aa 100755
--- a/hack/gantry-racer-bench/bench.sh
+++ b/hack/gantry-racer-bench/bench.sh
@@ -203,6 +203,7 @@ spec:
         run: $name
     spec:
 $(worker_affinity)
+      hostNetwork: true
       tolerations:
         - operator: Exists
       terminationGracePeriodSeconds: 1
diff --git a/hack/gantry-racer-bench/diskprep.sh b/hack/gantry-racer-bench/diskprep.sh
index acf0dd2ef..36c10a285 100644
--- a/hack/gantry-racer-bench/diskprep.sh
+++ b/hack/gantry-racer-bench/diskprep.sh
@@ -41,6 +41,22 @@ if [ -z "$disk" ]; then
 	echo "DISK $node none no-disk"
 	exec sleep infinity
 fi
+# Drop stale partition entries. This fails while any mount namespace still
+# holds the old /mnt filesystem, for example a pod with hostPath / created
+# before the unmount.
+b=$(basename "$disk")
+if ls "/sys/block/$b" | grep -q "^$b[0-9]"; then
+	blockdev --rereadpt "$disk" 2>/dev/null || partx -d "$disk" 2>/dev/null || true
+	udevadm settle 2>/dev/null || true
+fi
+if ls "/sys/block/$b" | grep -q "^$b[0-9]"; then
+	echo "DISK $node none partitions-busy"
+	exec sleep infinity
+fi
+if [ "$(head -c 1048576 "$disk" | tr -d '\0' | wc -c)" != 0 ]; then
+	echo "DISK $node none head-not-zero"
+	exec sleep infinity
+fi
 id=""
 for l in /dev/disk/by-id/wwn-* /dev/disk/by-id/scsi-3* /dev/disk/by-id/*; do
 	[ -e "$l" ] || continue

From dff742e53666045dbb4407e4afa0275c7594a7bc Mon Sep 17 00:00:00 2001
From: Jordan Olshevski <jordan.olshevski@microsoft.com>
Date: Wed, 28 Oct 2026 18:49:39 +0000
Subject: [PATCH 04/28] hack/gantry-racer-bench: metrics scrape and slab wipe
 helpers

---
 hack/gantry-racer-bench/scrape.sh     | 19 ++++++++++++
 hack/gantry-racer-bench/slabwipe.yaml | 43 +++++++++++++++++++++++++++
 2 files changed, 62 insertions(+)
 create mode 100755 hack/gantry-racer-bench/scrape.sh
 create mode 100644 hack/gantry-racer-bench/slabwipe.yaml

diff --git a/hack/gantry-racer-bench/scrape.sh b/hack/gantry-racer-bench/scrape.sh
new file mode 100755
index 000000000..9c9c2d977
--- /dev/null
+++ b/hack/gantry-racer-bench/scrape.sh
@@ -0,0 +1,19 @@
+#!/usr/bin/env bash
+# Sum Prometheus counters across all gantry (:9095) and racer-dataplane (:9090)
+# pods. Runs the scrape from the tools pod in gantry-bench.
+# Usage: scrape.sh <out-prefix>   -> <out-prefix>.gantry.prom, <out-prefix>.racer.prom
+set -euo pipefail
+out=$1
+ns=unbounded-system
+tools=$(kubectl -n gantry-bench get pod -l app=tools -o name | cut -d/ -f2)
+scrape() {
+  local label=$1 port=$2 file=$3
+  kubectl -n $ns get pods -l app.kubernetes.io/name=$label -o jsonpath='{range .items[*]}{.status.podIP}{"\n"}{end}' >"$file.ips"
+  kubectl -n gantry-bench cp "$file.ips" "$tools:/tmp/$label.ips" >/dev/null
+  kubectl -n gantry-bench exec "$tools" -- sh -c "rm -rf /tmp/m-$label; mkdir -p /tmp/m-$label; for ip in \$(cat /tmp/$label.ips); do wget -qO /tmp/m-$label/\$ip -T 5 http://\$ip:$port/metrics & if [ \$(jobs | wc -l) -ge 64 ]; then wait; fi; done; wait; cat /tmp/m-$label/*" 2>/dev/null |
+    grep -v '^#' | sed -E 's/(disk|worker|device)="[^"]*",?//g; s/\{\}//' | awk 'NF==2 {s[$1]+=$2} END {for (k in s) printf "%s %.0f\n", k, s[k]}' | sort >"$file"
+  rm -f "$file.ips"
+}
+scrape gantry 9095 "$out.gantry.prom" &
+scrape racer-dataplane 9090 "$out.racer.prom" &
+wait
diff --git a/hack/gantry-racer-bench/slabwipe.yaml b/hack/gantry-racer-bench/slabwipe.yaml
new file mode 100644
index 000000000..8f60c6e29
--- /dev/null
+++ b/hack/gantry-racer-bench/slabwipe.yaml
@@ -0,0 +1,43 @@
+# Workaround: racer-dataplane refuses to start (fatal InvalidConfiguration)
+# when slab files/checkpoints on disk were written with a different
+# RACER_SLAB_BYTES or worker count. This DaemonSet wipes the slab directory
+# once per node while no racer-dataplane process is running.
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: slabwipe
+  namespace: gantry-bench
+spec:
+  selector:
+    matchLabels: {app: slabwipe}
+  template:
+    metadata:
+      labels: {app: slabwipe}
+    spec:
+      hostPID: true
+      nodeSelector:
+        kubernetes.azure.com/agentpool: ddsv6
+      tolerations: [{operator: Exists}]
+      containers:
+      - name: wipe
+        image: mcr.microsoft.com/azurelinux/busybox:1.36
+        securityContext: {privileged: true}
+        env:
+        - name: MARK
+          value: "1"
+        command:
+        - chroot
+        - /host
+        - bash
+        - -c
+        - |
+          m=/var/lib/racer/.bench-wiped-$MARK
+          while [ ! -e $m ]; do
+            if ! pgrep -x racer-dataplane >/dev/null; then
+              rm -f /var/lib/racer/slabs/*; touch $m; echo "WIPED $(hostname)"; break
+            fi
+            sleep 1
+          done
+          sleep infinity
+        volumeMounts: [{name: host, mountPath: /host}]
+      volumes: [{name: host, hostPath: {path: /}}]

From 230a56a1b91c40f2fdab7804df1c20bf53f52be6 Mon Sep 17 00:00:00 2001
From: Jordan Olshevski <jordan.olshevski@microsoft.com>
Date: Wed, 28 Oct 2026 18:22:58 +0000
Subject: [PATCH 05/28] racer: allow slab files with stale sizes to be reset

Add an opt-in Slab::reset_mismatched_size that truncates and recreates a
regular slab file whose size differs from the configured capacity, after
validating every other dimension. Device-backed slabs are unaffected and
the default remains strict.
---
 cmd/racer-dataplane/alloc/src/slab.rs | 95 ++++++++++++++++++++++++++-
 1 file changed, 92 insertions(+), 3 deletions(-)

diff --git a/cmd/racer-dataplane/alloc/src/slab.rs b/cmd/racer-dataplane/alloc/src/slab.rs
index 4f1d7145c..c347e791d 100644
--- a/cmd/racer-dataplane/alloc/src/slab.rs
+++ b/cmd/racer-dataplane/alloc/src/slab.rs
@@ -64,6 +64,9 @@ pub struct Slab<C: Charge> {
 
     max_record_bytes: usize,
 
+    /// Recreate a regular slab file whose size differs from the configured capacity.
+    reset_mismatched: bool,
+
     opened: RefCell<Option<OpenSlab>>,
 
     writes: Rc<WriteState>,
@@ -84,12 +87,23 @@ impl<C: Charge> Slab<C> {
             capacity_bytes,
             segment_bytes,
             max_record_bytes,
+            reset_mismatched: false,
             opened: RefCell::new(None),
             writes: Rc::new(WriteState::default()),
             idle_buffer: Rc::new(RefCell::new(crate::BufferPool::default())),
         }
     }
 
+    /// Treat an existing regular slab file of the wrong size as a disposable cache:
+    /// truncate it to zero and recreate it at the configured capacity instead of
+    /// failing with `InvalidConfiguration`. Device-backed slabs ignore this flag.
+    /// The caller must ensure no recovered metadata references the old contents.
+    #[must_use]
+    pub fn reset_mismatched_size(mut self) -> Self {
+        self.reset_mismatched = true;
+        self
+    }
+
     /// Describe device ranges in logical segment order, without creating, sizing,
     /// or locking files. Bounds use BLKGETSIZE64 for block devices and file length
     /// for regular direct files. The caller must supply suitable alignment and keep
@@ -417,8 +431,16 @@ impl<C: Charge> Slab<C> {
             // The virtual filesystem has a fixed root owner, independent of host uid.
             validate_file(stat.stx_mode as u32, stat.stx_uid, 0, stat.stx_nlink as u64)?;
             let a = alignment_from_stat(&stat)?;
-            let geometry = self.validate_layout(a, stat.stx_size)?;
-            if stat.stx_size == 0 {
+            let mut size = stat.stx_size;
+            if self.reset_mismatched && size != 0 && size != self.capacity_bytes {
+                self.validate_layout(a, 0)?;
+                handle
+                    .set_len(0)
+                    .map_err(|e| system_error("ftruncate", e))?;
+                size = 0;
+            }
+            let geometry = self.validate_layout(a, size)?;
+            if size == 0 {
                 handle
                     .set_len(self.capacity_bytes)
                     .map_err(|e| system_error("ftruncate", e))?;
@@ -449,7 +471,7 @@ impl<C: Charge> Slab<C> {
             unsafe { libc::geteuid() },
             metadata.nlink(),
         )?;
-        let size = metadata.len();
+        let mut size = metadata.len();
         // Delay O_DIRECT until after rejecting nonregular files (including FIFOs).
         // SAFETY: fcntl synchronously borrows this live descriptor.
         let flags = unsafe { libc::fcntl(file.as_raw_fd(), libc::F_GETFL) };
@@ -470,6 +492,17 @@ impl<C: Charge> Slab<C> {
             ));
         }
         let a = probe(&file)?;
+        if self.reset_mismatched && size != 0 && size != self.capacity_bytes {
+            // Validate everything except the stale size before discarding data.
+            self.validate_layout(a, 0)?;
+            eprintln!(
+                "racer: slab {} has {size} bytes, configured {}; discarding stale cache contents",
+                path.display(),
+                self.capacity_bytes
+            );
+            file.set_len(0).map_err(|e| system_error("ftruncate", e))?;
+            size = 0;
+        }
         let geometry = self.validate_layout(a, size)?;
         if size == 0 {
             file.set_len(self.capacity_bytes)
@@ -1871,6 +1904,62 @@ mod tests {
         assert_eq!(std::fs::metadata(path).unwrap().len(), capacity);
     }
 
+    /// Opt-in reset recreates stale-size cache files but still rejects invalid dimensions.
+    #[test]
+    fn reset_mismatched_size_recreates_grown_or_shrunk_files() {
+        use std::os::unix::fs::PermissionsExt;
+        let directory = Directory::new();
+        let probe = Slab::<()>::new(directory.0.join("capability-probe"), 4096, 4096, 512);
+        if real_alignment(probe.open_now()).is_none() {
+            return;
+        }
+        for (old, new) in [(8192u64, 16384u64), (16384, 8192)] {
+            let path = directory.0.join(format!("data-{old}-{new}"));
+            std::fs::write(&path, vec![42u8; old as usize]).unwrap();
+            std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o600)).unwrap();
+            // Without opt-in the stale size is still rejected and preserved.
+            let strict = Slab::<()>::new(path.clone(), new, 4096, 512);
+            assert_eq!(strict.open_now(), Err(Error::InvalidConfiguration));
+            assert_eq!(std::fs::metadata(&path).unwrap().len(), old);
+            drop(strict);
+            let slab = Slab::<()>::new(path.clone(), new, 4096, 512).reset_mismatched_size();
+            assert!(real_alignment(slab.open_now()).is_some());
+            assert_eq!(slab.capacity_bytes(), new);
+            drop(slab);
+            let contents = std::fs::read(&path).unwrap();
+            assert_eq!(contents.len() as u64, new);
+            // Stale cache bytes are discarded rather than reinterpreted.
+            assert!(contents.iter().all(|b| *b == 0));
+        }
+        // Matching sizes keep their contents.
+        let path = directory.0.join("same");
+        std::fs::write(&path, vec![7u8; 8192]).unwrap();
+        std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o600)).unwrap();
+        let slab = Slab::<()>::new(path.clone(), 8192, 4096, 512).reset_mismatched_size();
+        assert!(real_alignment(slab.open_now()).is_some());
+        drop(slab);
+        assert!(std::fs::read(&path).unwrap().iter().all(|b| *b == 7));
+        // Genuinely invalid configurations still fail and leave stale files untouched.
+        let path = directory.0.join("invalid-existing");
+        std::fs::write(&path, vec![9u8; 8192]).unwrap();
+        std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o600)).unwrap();
+        for (capacity, segment, record) in [
+            (0, 4096, 512),
+            (4096, 0, 512),
+            (4096, 4096, 0),
+            (12289, 4096, 512),
+            (16384, 4096, 4097),
+        ] {
+            assert_eq!(
+                Slab::<()>::new(path.clone(), capacity, segment, record)
+                    .reset_mismatched_size()
+                    .open_now(),
+                Err(Error::InvalidConfiguration)
+            );
+            assert_eq!(std::fs::read(&path).unwrap(), vec![9u8; 8192]);
+        }
+    }
+
     /// Invalid startup dimensions never truncate a preexisting nonempty file.
     #[test]
     fn open_rejects_bad_layout_and_existing_size_without_truncating() {

From 740903af235ee792fd806bd57a2fba1d88280b6d Mon Sep 17 00:00:00 2001
From: Jordan Olshevski <jordan.olshevski@microsoft.com>
Date: Wed, 28 Oct 2026 18:25:29 +0000
Subject: [PATCH 06/28] racer: cold start on slab geometry changes in slab-file
 mode

Open per-worker slab files with reset_mismatched_size so a changed
RACER_SLAB_BYTES recreates the file instead of failing startup with
InvalidConfiguration. Checkpoints recorded for the old geometry or worker
set are already skipped by recovery, so the node starts cold. Remove slab
files left by workers outside the current plan. Device mode is unchanged.
---
 cmd/racer-dataplane/src/app.rs          | 43 ++++++++++++++++++++++
 cmd/racer-dataplane/src/app/recovery.rs | 49 +++++++++++++++++++++++++
 cmd/racer-dataplane/src/app/tests.rs    | 42 +++++++++++++++++++++
 3 files changed, 134 insertions(+)

diff --git a/cmd/racer-dataplane/src/app.rs b/cmd/racer-dataplane/src/app.rs
index 85fb4215f..3efb034c8 100644
--- a/cmd/racer-dataplane/src/app.rs
+++ b/cmd/racer-dataplane/src/app.rs
@@ -243,6 +243,41 @@ impl NodeState {
     }
 }
 
+/// Remove slab files left by workers that no longer exist after a worker-count change.
+///
+/// Only regular files named `worker-<id>-slab-0.dat` with an id outside `workers` are
+/// removed. Failures are logged and ignored because stale files never affect correctness.
+fn prune_stale_slab_files(directory: &std::path::Path, workers: &[WorkerId]) {
+    let Ok(entries) = std::fs::read_dir(directory) else {
+        return;
+    };
+    for entry in entries.flatten() {
+        let name = entry.file_name();
+        let Some(id) = name
+            .to_str()
+            .and_then(|n| n.strip_prefix("worker-"))
+            .and_then(|n| n.strip_suffix("-slab-0.dat"))
+            .and_then(|n| n.parse::<u16>().ok())
+        else {
+            continue;
+        };
+        if workers.contains(&WorkerId(id)) || !entry.file_type().is_ok_and(|t| t.is_file()) {
+            continue;
+        }
+        let path = entry.path();
+        match std::fs::remove_file(&path) {
+            Ok(()) => eprintln!(
+                "racer-dataplane: stage=slabs removed stale slab file {}",
+                path.display()
+            ),
+            Err(error) => eprintln!(
+                "racer-dataplane: stage=slabs failed to remove stale slab file {}: {error}",
+                path.display()
+            ),
+        }
+    }
+}
+
 impl Application {
     /// Composition only. Does not open files, spawn threads, or accept requests.
     pub fn assemble(config: Config) -> Result<Self> {
@@ -346,6 +381,10 @@ impl Application {
                 .map(devices::Plan::claimed_disks)
                 .unwrap_or_default(),
         });
+        if devices.is_none() {
+            let workers: Vec<_> = plan.pairs.iter().map(|p| p.worker).collect();
+            prune_stale_slab_files(&self.config.slab_directory, &workers);
+        }
         node.devices = devices;
         Arc::get_mut(&mut self.config)
             .ok_or(Error::InvalidConfiguration)?
@@ -1491,6 +1530,10 @@ impl WorkerApplication {
                 config.segment_bytes,
                 crate::model::PAGE_BYTES as usize + crate::store::MAX_HEADER_BYTES + 16,
             )
+            // Slab files are a re-fetchable cache. A changed slab size makes any
+            // checkpoint for this file incompatible (its recorded slab_bytes no
+            // longer match), so recovery starts this worker cold.
+            .reset_mismatched_size()
         });
         let reader = Rc::new(
             StoreReader::new(
diff --git a/cmd/racer-dataplane/src/app/recovery.rs b/cmd/racer-dataplane/src/app/recovery.rs
index 53eaf179b..6b13ca3a1 100644
--- a/cmd/racer-dataplane/src/app/recovery.rs
+++ b/cmd/racer-dataplane/src/app/recovery.rs
@@ -1001,6 +1001,55 @@ mod tests {
         );
     }
 
+    /// A geometry change (slab size, segment size, or worker count) skips stale
+    /// checkpoints and starts cold instead of failing node recovery.
+    #[test]
+    fn changed_slab_size_or_worker_count_skips_stale_checkpoint() {
+        let node = NodeState::default();
+        let keys = crate::test_support::security::keys();
+        let live = |ids: &[WorkerId], g: CheckpointGeometry| -> HashMap<_, _> {
+            ids.iter().map(|id| (*id, g)).collect()
+        };
+        let choose = |cuts: Vec<CheckpointImage>, g: &HashMap<WorkerId, CheckpointGeometry>| {
+            select(cuts, g, &node.workers, (4, 4), &keys, &caches())
+        };
+        let three = [WorkerId(0), WorkerId(1), WorkerId(2)];
+        let five = [
+            WorkerId(0),
+            WorkerId(1),
+            WorkerId(2),
+            WorkerId(3),
+            WorkerId(4),
+        ];
+        // Sanity: an unchanged geometry and worker set recovers.
+        assert!(choose(vec![image(1, &three)], &live(&three, geometry())).is_some());
+        // Larger slab per worker.
+        let bigger =
+            CheckpointGeometry::new(16384, 4096, 4, Alignment::new(4096, 4096, 4096).unwrap())
+                .unwrap();
+        assert!(choose(vec![image(1, &three)], &live(&three, bigger)).is_none());
+        // Different segment size.
+        let segment =
+            CheckpointGeometry::new(8192, 8192, 1, Alignment::new(4096, 4096, 4096).unwrap())
+                .unwrap();
+        assert!(choose(vec![image(1, &three)], &live(&three, segment)).is_none());
+        // More workers, with and without a slab size change.
+        assert!(choose(vec![image(1, &three)], &live(&five, geometry())).is_none());
+        assert!(choose(vec![image(1, &three)], &live(&five, bigger)).is_none());
+        // Fewer workers.
+        assert!(choose(vec![image(1, &five)], &live(&three, geometry())).is_none());
+        // A newer compatible cut still wins over older stale ones.
+        assert_eq!(
+            choose(
+                vec![image(3, &five), image(2, &three)],
+                &live(&three, geometry())
+            )
+            .unwrap()
+            .sequence,
+            2
+        );
+    }
+
     #[test]
     fn newest_complete_cut_wins_and_partial_duplicate_or_foreign_workers_fall_back() {
         let node = NodeState::default();
diff --git a/cmd/racer-dataplane/src/app/tests.rs b/cmd/racer-dataplane/src/app/tests.rs
index 84a83f58b..bb656fa81 100644
--- a/cmd/racer-dataplane/src/app/tests.rs
+++ b/cmd/racer-dataplane/src/app/tests.rs
@@ -4880,3 +4880,45 @@ fn snapshot_refresh_applies_priority_live_and_reweighs_memory() {
         );
     }
 }
+
+/// Slab files for workers outside the current plan are removed; others are kept.
+#[test]
+fn prune_stale_slab_files_removes_only_extra_worker_files() {
+    let directory = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR"))
+        .join("target")
+        .join(format!("prune-slabs-test-{}", std::process::id()));
+    let _ = std::fs::remove_dir_all(&directory);
+    std::fs::create_dir_all(&directory).unwrap();
+    for name in [
+        "worker-0-slab-0.dat",
+        "worker-1-slab-0.dat",
+        "worker-2-slab-0.dat",
+        "worker-3-slab-0.dat",
+        "worker-4-slab-0.dat",
+        "checkpoint.0",
+        "worker-x-slab-0.dat",
+    ] {
+        std::fs::write(directory.join(name), b"x").unwrap();
+    }
+    std::fs::create_dir(directory.join("worker-9-slab-0.dat")).unwrap();
+    super::prune_stale_slab_files(&directory, &[WorkerId(0), WorkerId(1), WorkerId(2)]);
+    let mut left: Vec<_> = std::fs::read_dir(&directory)
+        .unwrap()
+        .map(|e| e.unwrap().file_name().into_string().unwrap())
+        .collect();
+    left.sort();
+    assert_eq!(
+        left,
+        [
+            "checkpoint.0",
+            "worker-0-slab-0.dat",
+            "worker-1-slab-0.dat",
+            "worker-2-slab-0.dat",
+            "worker-9-slab-0.dat",
+            "worker-x-slab-0.dat",
+        ]
+    );
+    // A missing directory is not an error.
+    super::prune_stale_slab_files(&directory.join("missing"), &[WorkerId(0)]);
+    std::fs::remove_dir_all(&directory).unwrap();
+}

From bce4275f56aec36f34411321fd5db1661b16b4d2 Mon Sep 17 00:00:00 2001
From: Jordan Olshevski <jordan.olshevski@microsoft.com>
Date: Wed, 28 Oct 2026 18:45:15 +0000
Subject: [PATCH 07/28] gantry/racer: retry transient Racer failures and resume
 mid-body

Retry Stat and Get on racersdk.ErrUnavailable with jittered exponential
backoff bounded by a per-request budget and the request context. After
headers are committed, resume a failed WriteTo from the exact byte count
accepted by the response, pinned to the original ETag, and abort if the
resumed metadata differs. Add gantry_racer_mirror_retries_total{stage}.
---
 internal/gantry/racer/racer.go      | 208 ++++++++++++-
 internal/gantry/racer/retry_test.go | 462 ++++++++++++++++++++++++++++
 2 files changed, 656 insertions(+), 14 deletions(-)
 create mode 100644 internal/gantry/racer/retry_test.go

diff --git a/internal/gantry/racer/racer.go b/internal/gantry/racer/racer.go
index 7b4e4b67d..a263ce43e 100644
--- a/internal/gantry/racer/racer.go
+++ b/internal/gantry/racer/racer.go
@@ -7,12 +7,13 @@ package racer
 
 import (
 	"context"
-	"crypto/rand"
+	crand "crypto/rand"
 	"encoding/json"
 	"errors"
 	"fmt"
 	"io"
 	"log/slog"
+	"math/rand/v2"
 	"net"
 	"net/http"
 	"net/url"
@@ -417,15 +418,39 @@ type Handler struct {
 	auth        authenticationChallenger
 	logger      *slog.Logger
 	diagnostics racerFailureDiagnostics
+	retry       retryPolicy
+}
+
+// HandlerOption configures optional Handler behavior.
+type HandlerOption func(*Handler)
+
+// WithRetryBudget bounds the total time one request may spend backing off
+// between retries of transient Racer failures. Zero disables retries.
+func WithRetryBudget(budget time.Duration) HandlerOption {
+	return func(h *Handler) { h.retry.budget = max(budget, 0) }
+}
+
+// WithRetryObserver reports each retry by stage (stat, get, or write_body).
+// observe must be concurrency-safe and must not block.
+func WithRetryObserver(observe func(stage string)) HandlerOption {
+	return func(h *Handler) { h.retry.observe = observe }
 }
 
 // NewHandler preserves the mirror's logging context and optional auth challenges.
-func NewHandler(client Client, auth authenticationChallenger, logger *slog.Logger) *Handler {
+func NewHandler(client Client, auth authenticationChallenger, logger *slog.Logger, opts ...HandlerOption) *Handler {
 	if logger == nil {
 		logger = slog.Default()
 	}
 
-	return &Handler{client: client, auth: auth, logger: logger.With(slog.String("subsystem", "mirror"))}
+	h := &Handler{
+		client: client, auth: auth, logger: logger.With(slog.String("subsystem", "mirror")),
+		retry: retryPolicy{budget: handlerDefaultRetryBudget, base: defaultRetryBase, cap: defaultRetryCap},
+	}
+	for _, opt := range opts {
+		opt(h)
+	}
+
+	return h
 }
 
 // ServeContent serves a validated reference. Positive blob offsets select a
@@ -435,7 +460,7 @@ func (s *Handler) ServeContent(w http.ResponseWriter, r *http.Request, ref iface
 	writeRacerDownloadHeaders(w)
 
 	// Fresh opaque correlation, including for aborts after headers are committed.
-	id := rand.Text()
+	id := crand.Text()
 	w.Header().Set("Gantry-Racer-Request-ID", id)
 
 	if s.client == nil {
@@ -467,7 +492,9 @@ func (s *Handler) ServeContent(w http.ResponseWriter, r *http.Request, ref iface
 		return
 	}
 
-	value, err := s.client.Get(r.Context(), request, options...)
+	retry := s.retry.start()
+
+	value, err := s.getWithRetry(r.Context(), retry, request, options...)
 	if err != nil {
 		expected := int64(-1)
 		if ranged {
@@ -480,7 +507,7 @@ func (s *Handler) ServeContent(w http.ResponseWriter, r *http.Request, ref iface
 		return
 	}
 
-	defer func() { _ = value.Close() }() //nolint:errcheck // best-effort close
+	defer func() { _ = value.Close() }() //nolint:errcheck // best-effort close of the first read; resumes close their own
 
 	actual := value.Metadata()
 	// The manifest route also serves indexes, so an absent media type is ambiguous.
@@ -504,7 +531,8 @@ func (s *Handler) ServeContent(w http.ResponseWriter, r *http.Request, ref iface
 		actual = metadata
 	}
 
-	s.streamContent(w, id, ref, actual, offset, value)
+	smallObject := len(options) != 0 && options[0].SmallObject
+	s.streamContent(r.Context(), w, id, ref, request, actual, offset, value, retry, smallObject)
 }
 
 // prepareResponse handles HEAD and validates the snapshot needed for blob resume.
@@ -519,7 +547,7 @@ func (s *Handler) prepareResponse(w http.ResponseWriter, r *http.Request, id str
 		return racersdk.Metadata{}, nil, false
 	}
 
-	metadata, err := s.client.Stat(r.Context(), request)
+	metadata, err := s.statWithRetry(r.Context(), s.retry.start(), request)
 	if err != nil {
 		s.logRacerFailure(id, "stat", err, -1, 0)
 		s.racerError(w, r, ref, err)
@@ -554,7 +582,7 @@ func (s *Handler) prepareResponse(w http.ResponseWriter, r *http.Request, id str
 	return metadata, options, false
 }
 
-func (s *Handler) streamContent(w http.ResponseWriter, id string, ref ifaces.OriginRef, metadata racersdk.Metadata, offset int64, value *racersdk.Object) {
+func (s *Handler) streamContent(ctx context.Context, w http.ResponseWriter, id string, ref ifaces.OriginRef, request racersdk.Request, metadata racersdk.Metadata, offset int64, value *racersdk.Object, retry *retryState, smallObject bool) {
 	size := int64(metadata.Size)
 	writeRacerHeaders(w, ref.Digest, metadata, ref.Kind)
 
@@ -578,12 +606,151 @@ func (s *Handler) streamContent(w http.ResponseWriter, id string, ref ifaces.Ori
 		}
 	}
 	// The SDK gates the final byte on Complete, including for resumed ranges.
-	// Earlier incomplete page prefixes may already be visible on failure: abort
-	// rather than append an error body or let net/http complete the response.
+	// Earlier incomplete page prefixes may already be visible on failure.
+	// Transient Racer failures resume the same response from the exact count
+	// of bytes accepted by w, pinned to the same version; every resumed read
+	// again gates its final byte on Complete. Otherwise abort rather than
+	// append an error body or let net/http complete the response.
 	// OCI digest verification belongs to the consumer's assembled object.
-	if n, err := value.WriteTo(w); err != nil || n != remaining {
-		s.logRacerFailure(id, "write_body", err, remaining, n)
-		panic(http.ErrAbortHandler)
+	var written int64
+
+	current := value
+
+	defer func() {
+		if current != nil && current != value {
+			_ = current.Close() //nolint:errcheck // best-effort close of a resumed read
+		}
+	}()
+
+	for {
+		n, err := current.WriteTo(w)
+		written += n
+
+		if err == nil && written == remaining {
+			return
+		}
+
+		if err == nil || written > remaining || !retryable(ctx, err) || !retry.wait(ctx, "write_body") {
+			s.logRacerFailure(id, "write_body", err, remaining, written)
+			panic(http.ErrAbortHandler)
+		}
+
+		// WriteTo may leave an interrupt deadline behind; clear it before
+		// resuming into the same response.
+		if err := http.NewResponseController(w).SetWriteDeadline(time.Time{}); err != nil && !errors.Is(err, http.ErrNotSupported) {
+			s.logRacerFailure(id, "write_body", err, remaining, written)
+			panic(http.ErrAbortHandler)
+		}
+
+		if current != value {
+			_ = current.Close() //nolint:errcheck // best-effort close before resume
+		}
+
+		current = nil
+		options := racersdk.ReadOptions{Offset: offset + written, ETag: metadata.ETag, SmallObject: smallObject}
+
+		next, err := s.getWithRetry(ctx, retry, request, options)
+		if err != nil {
+			s.logRacerFailure(id, "resume_get", err, remaining, written)
+			panic(http.ErrAbortHandler)
+		}
+
+		current = next
+
+		if actual := next.Metadata(); actual.Size != metadata.Size || actual.ETag != metadata.ETag || actual.ContentType != metadata.ContentType {
+			s.logRacerFailure(id, "resume_metadata", errors.New("invalid metadata"), remaining, written)
+			panic(http.ErrAbortHandler)
+		}
+	}
+}
+
+// Bounded retry of transient Racer failures.
+
+const (
+	// DefaultRetryBudget bounds per-request backoff when no budget is configured.
+	DefaultRetryBudget = 30 * time.Second
+	defaultRetryBase   = 150 * time.Millisecond
+	defaultRetryCap    = 2 * time.Second
+)
+
+// handlerDefaultRetryBudget is a variable only so tests of single-attempt
+// failure paths can disable retries package-wide.
+var handlerDefaultRetryBudget = DefaultRetryBudget
+
+type retryPolicy struct {
+	budget, base, cap time.Duration
+	observe           func(stage string)
+}
+
+// retryState tracks one request's retry budget across all stages.
+type retryState struct {
+	policy   retryPolicy
+	deadline time.Time
+	attempt  int
+}
+
+func (p retryPolicy) start() *retryState {
+	return &retryState{policy: p, deadline: time.Now().Add(p.budget)}
+}
+
+// wait sleeps for a jittered exponential backoff and reports whether the
+// caller should retry. It refuses when the sleep would exceed the budget or
+// the request context ends.
+func (r *retryState) wait(ctx context.Context, stage string) bool {
+	if r.policy.budget <= 0 || ctx.Err() != nil {
+		return false
+	}
+
+	backoff := r.policy.cap
+	if r.attempt < 30 {
+		backoff = min(r.policy.base<<r.attempt, r.policy.cap)
+	}
+
+	r.attempt++
+	// Equal jitter keeps a floor while spreading synchronized retries.
+	delay := backoff/2 + rand.N(backoff/2+1) //nolint:gosec // jitter, not security
+
+	if time.Now().Add(delay).After(r.deadline) {
+		return false
+	}
+
+	if r.policy.observe != nil {
+		r.policy.observe(stage)
+	}
+
+	timer := time.NewTimer(delay)
+	defer timer.Stop()
+
+	select {
+	case <-ctx.Done():
+		return false
+	case <-timer.C:
+		return true
+	}
+}
+
+// retryable reports whether err is a transient Racer failure worth retrying.
+// Destination failures and ambiguous write timeouts are not: the downstream
+// client may be at fault. Caller cancellation always stops retries.
+func retryable(ctx context.Context, err error) bool {
+	return ctx.Err() == nil && !errors.Is(err, racersdk.ErrDestination) && errors.Is(err, racersdk.ErrUnavailable)
+}
+
+func (s *Handler) getWithRetry(ctx context.Context, retry *retryState, request racersdk.Request, options ...racersdk.ReadOptions) (*racersdk.Object, error) {
+	for {
+		value, err := s.client.Get(ctx, request, options...)
+		if err == nil || !retryable(ctx, err) || !retry.wait(ctx, "get") {
+			return value, err
+		}
+	}
+}
+
+func (s *Handler) statWithRetry(ctx context.Context, retry *retryState, request racersdk.Request) (racersdk.Metadata, error) {
+	for {
+		metadata, err := s.client.Stat(ctx, request)
+		if err == nil || !retryable(ctx, err) || !retry.wait(ctx, "stat") {
+			return metadata, err
+		}
 	}
 }
 
@@ -1115,6 +1282,7 @@ type Metrics struct {
 	durations       *prometheus.HistogramVec
 	originRequests  *prometheus.CounterVec
 	originBodyBytes *prometheus.CounterVec
+	retries         *prometheus.CounterVec
 }
 
 // NewMetrics registers mirror and origin metrics.
@@ -1125,6 +1293,7 @@ func NewMetrics(reg *metrics.Registry) *Metrics {
 		durations:       reg.NewHistogramVec("racer", prometheus.HistogramOpts{Name: "gantry_racer_mirror_duration_seconds", Help: "Racer mirror handler duration including downstream writes.", Buckets: prometheus.ExponentialBuckets(0.001, 4, 11)}, []string{"method"}),
 		originRequests:  reg.NewCounterVec("racer", prometheus.CounterOpts{Name: "gantry_racer_origin_requests_total", Help: "Upstream HTTP round trips, including HEAD, GET, and authentication; status zero denotes transport failure."}, []string{"method", "status"}),
 		originBodyBytes: reg.NewCounterVec("racer", prometheus.CounterOpts{Name: "gantry_racer_origin_bytes_total", Help: "Upstream GET body bytes read, including partial transfers."}, []string{"kind"}),
+		retries:         reg.NewCounterVec("racer", prometheus.CounterOpts{Name: "gantry_racer_mirror_retries_total", Help: "Racer mirror retries of transient failures by stage (stat, get, write_body)."}, []string{"stage"}),
 	}
 
 	return m
@@ -1162,3 +1331,14 @@ func (m *Metrics) OriginRequest(method string, status int) {
 func (m *Metrics) OriginBytes(kind string, bytes int64) {
 	m.originBodyBytes.WithLabelValues(kind).Add(float64(bytes))
 }
+
+// MirrorRetry records one retry of a transient Racer failure.
+func (m *Metrics) MirrorRetry(stage string) {
+	switch stage {
+	case "stat", "get", "write_body":
+	default:
+		stage = "other"
+	}
+
+	m.retries.WithLabelValues(stage).Inc()
+}
diff --git a/internal/gantry/racer/retry_test.go b/internal/gantry/racer/retry_test.go
new file mode 100644
index 000000000..7a7e26a6c
--- /dev/null
+++ b/internal/gantry/racer/retry_test.go
@@ -0,0 +1,462 @@
+// Copyright (c) Microsoft Corporation.
+// SPDX-License-Identifier: Apache-2.0
+
+package racer
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"sync"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"github.com/Azure/unbounded/internal/gantry/digest"
+	"github.com/Azure/unbounded/internal/gantry/ifaces"
+	"github.com/Azure/unbounded/internal/gantry/mirror"
+	"github.com/Azure/unbounded/pkg/racersdk"
+)
+
+// Most handler tests exercise single-attempt failure paths. Retry tests opt
+// in explicitly with WithRetryBudget.
+func TestMain(m *testing.M) {
+	handlerDefaultRetryBudget = 0
+
+	os.Exit(m.Run())
+}
+
+// retryClient injects failures before delegating to a real fake SDK client.
+type retryClient struct {
+	Client
+	mu        sync.Mutex
+	gets      int
+	stats     int
+	getErr    func(call int, opts []racersdk.ReadOptions) error
+	statErr   func(call int) error
+	getHook   func(ctx context.Context, call int, req racersdk.Request, opts []racersdk.ReadOptions) (*racersdk.Object, error, bool)
+	afterCall func()
+}
+
+func (c *retryClient) Get(ctx context.Context, req racersdk.Request, opts ...racersdk.ReadOptions) (*racersdk.Object, error) {
+	c.mu.Lock()
+	c.gets++
+	call := c.gets
+	c.mu.Unlock()
+
+	if c.afterCall != nil {
+		defer c.afterCall()
+	}
+
+	if c.getErr != nil {
+		if err := c.getErr(call, opts); err != nil {
+			return nil, err
+		}
+	}
+
+	if c.getHook != nil {
+		if value, err, ok := c.getHook(ctx, call, req, opts); ok {
+			return value, err
+		}
+	}
+
+	return c.Client.Get(ctx, req, opts...)
+}
+
+func (c *retryClient) Stat(ctx context.Context, req racersdk.Request) (racersdk.Metadata, error) {
+	c.mu.Lock()
+	c.stats++
+	call := c.stats
+	c.mu.Unlock()
+
+	if c.statErr != nil {
+		if err := c.statErr(call); err != nil {
+			return racersdk.Metadata{}, err
+		}
+	}
+
+	return c.Client.Stat(ctx, req)
+}
+
+type retryStages struct {
+	mu     sync.Mutex
+	stages map[string]int
+}
+
+func (r *retryStages) observe(stage string) {
+	r.mu.Lock()
+	defer r.mu.Unlock()
+
+	if r.stages == nil {
+		r.stages = map[string]int{}
+	}
+
+	r.stages[stage]++
+}
+
+func (r *retryStages) count(stage string) int {
+	r.mu.Lock()
+	defer r.mu.Unlock()
+
+	return r.stages[stage]
+}
+
+func retryHandler(client Client, budget time.Duration, stages *retryStages) *Handler {
+	h := NewHandler(client, nil, nil, WithRetryBudget(budget), WithRetryObserver(stages.observe))
+	h.retry.base = time.Millisecond
+	h.retry.cap = 5 * time.Millisecond
+
+	return h
+}
+
+// serveRetry runs ServeContent and reports whether the handler aborted.
+func serveRetry(h *Handler, r *http.Request, ref ifaces.OriginRef) (rec *httptest.ResponseRecorder, aborted bool) {
+	rec = httptest.NewRecorder()
+
+	defer func() {
+		if p := recover(); p != nil {
+			if p != http.ErrAbortHandler {
+				panic(p)
+			}
+
+			aborted = true
+		}
+	}()
+
+	h.ServeContent(rec, r, ref)
+
+	return rec, false
+}
+
+func retryRef(d digest.Digest) ifaces.OriginRef {
+	return ifaces.OriginRef{Registry: "registry.example", Repository: "repo", Digest: d, Kind: ifaces.KindBlob}
+}
+
+func unavailable() error { return fmt.Errorf("overloaded: %w", racersdk.ErrUnavailable) }
+
+func staticOrigin(t *testing.T, data []byte) racersdk.Origin {
+	t.Helper()
+
+	metadata := racerMetadata(t, racerDigest(data), len(data))
+
+	return func(_ context.Context, req racersdk.OriginRequest) (racersdk.Metadata, io.ReadCloser, error) {
+		if req.Head {
+			return metadata, nil, nil
+		}
+
+		return metadata, io.NopCloser(bytes.NewReader(data[req.Offset:])), nil
+	}
+}
+
+func TestRacerRetryGetTransientThenSuccess(t *testing.T) {
+	data := []byte("abc")
+	client := &retryClient{
+		Client: racerFakeClient(t, staticOrigin(t, data)),
+		getErr: func(call int, _ []racersdk.ReadOptions) error {
+			if call <= 2 {
+				return unavailable()
+			}
+
+			return nil
+		},
+	}
+
+	var stages retryStages
+
+	rec, aborted := serveRetry(retryHandler(client, 5*time.Second, &stages), httptest.NewRequest(http.MethodGet, "/", nil), retryRef(racerDigest(data)))
+	if aborted || rec.Code != http.StatusOK || rec.Body.String() != "abc" {
+		t.Fatalf("aborted=%v status=%d body=%q", aborted, rec.Code, rec.Body.String())
+	}
+
+	if client.gets != 3 || stages.count("get") != 2 {
+		t.Fatalf("gets=%d retries=%d", client.gets, stages.count("get"))
+	}
+}
+
+func TestRacerRetryStatTransientThenSuccess(t *testing.T) {
+	data := []byte("abc")
+	client := &retryClient{
+		Client: racerFakeClient(t, staticOrigin(t, data)),
+		statErr: func(call int) error {
+			if call == 1 {
+				return unavailable()
+			}
+
+			return nil
+		},
+	}
+
+	var stages retryStages
+
+	rec, aborted := serveRetry(retryHandler(client, 5*time.Second, &stages), httptest.NewRequest(http.MethodHead, "/", nil), retryRef(racerDigest(data)))
+	if aborted || rec.Code != http.StatusOK || rec.Header().Get("Content-Length") != "3" {
+		t.Fatalf("aborted=%v status=%d headers=%v", aborted, rec.Code, rec.Header())
+	}
+
+	if client.stats != 2 || stages.count("stat") != 1 {
+		t.Fatalf("stats=%d retries=%d", client.stats, stages.count("stat"))
+	}
+}
+
+func TestRacerRetryGivesUpAfterBudget(t *testing.T) {
+	data := []byte("abc")
+	client := &retryClient{
+		Client: racerFakeClient(t, staticOrigin(t, data)),
+		getErr: func(int, []racersdk.ReadOptions) error { return unavailable() },
+	}
+
+	var stages retryStages
+
+	started := time.Now()
+
+	rec, aborted := serveRetry(retryHandler(client, 50*time.Millisecond, &stages), httptest.NewRequest(http.MethodGet, "/", nil), retryRef(racerDigest(data)))
+	if aborted || rec.Code != http.StatusServiceUnavailable {
+		t.Fatalf("aborted=%v status=%d", aborted, rec.Code)
+	}
+
+	if elapsed := time.Since(started); elapsed > 2*time.Second {
+		t.Fatalf("exceeded budget: %v", elapsed)
+	}
+
+	if client.gets < 2 || stages.count("get") != client.gets-1 {
+		t.Fatalf("gets=%d retries=%d", client.gets, stages.count("get"))
+	}
+}
+
+func TestRacerRetryDisabledWithZeroBudget(t *testing.T) {
+	data := []byte("abc")
+	client := &retryClient{
+		Client: racerFakeClient(t, staticOrigin(t, data)),
+		getErr: func(int, []racersdk.ReadOptions) error { return unavailable() },
+	}
+
+	var stages retryStages
+
+	rec, _ := serveRetry(retryHandler(client, 0, &stages), httptest.NewRequest(http.MethodGet, "/", nil), retryRef(racerDigest(data)))
+	if rec.Code != http.StatusServiceUnavailable || client.gets != 1 || stages.count("get") != 0 {
+		t.Fatalf("status=%d gets=%d", rec.Code, client.gets)
+	}
+}
+
+func TestRacerRetryNotOnPermanentErrors(t *testing.T) {
+	for _, tc := range []struct {
+		err    error
+		status int
+	}{
+		{racersdk.ErrNotFound, http.StatusNotFound},
+		{racersdk.ErrUnauthorized, http.StatusUnauthorized},
+		{racersdk.ErrForbidden, http.StatusForbidden},
+		{fmt.Errorf("%w: %w", racersdk.ErrDestination, racersdk.ErrUnavailable), http.StatusServiceUnavailable},
+	} {
+		t.Run(tc.err.Error(), func(t *testing.T) {
+			data := []byte("abc")
+			client := &retryClient{
+				Client: racerFakeClient(t, staticOrigin(t, data)),
+				getErr: func(int, []racersdk.ReadOptions) error { return tc.err },
+			}
+
+			var stages retryStages
+
+			rec, _ := serveRetry(retryHandler(client, 5*time.Second, &stages), httptest.NewRequest(http.MethodGet, "/", nil), retryRef(racerDigest(data)))
+			if rec.Code != tc.status || client.gets != 1 || stages.count("get") != 0 {
+				t.Fatalf("status=%d gets=%d retries=%d", rec.Code, client.gets, stages.count("get"))
+			}
+		})
+	}
+}
+
+func TestRacerRetryStopsOnClientCancel(t *testing.T) {
+	data := []byte("abc")
+	ctx, cancel := context.WithCancel(context.Background())
+
+	client := &retryClient{
+		Client:    racerFakeClient(t, staticOrigin(t, data)),
+		getErr:    func(int, []racersdk.ReadOptions) error { return unavailable() },
+		afterCall: cancel,
+	}
+
+	var stages retryStages
+
+	h := retryHandler(client, 10*time.Second, &stages)
+	h.retry.base = time.Second
+	h.retry.cap = time.Second
+	started := time.Now()
+
+	rec, _ := serveRetry(h, httptest.NewRequest(http.MethodGet, "/", nil).WithContext(ctx), retryRef(racerDigest(data)))
+	if rec.Code != http.StatusServiceUnavailable || client.gets != 1 || time.Since(started) > 500*time.Millisecond {
+		t.Fatalf("status=%d gets=%d elapsed=%v", rec.Code, client.gets, time.Since(started))
+	}
+}
+
+// truncatingOrigin serves data, but the first body read fails at cut.
+func truncatingOrigin(t *testing.T, data []byte, cut int, calls *atomic.Int32) racersdk.Origin {
+	t.Helper()
+
+	metadata := racerMetadata(t, racerDigest(data), len(data))
+
+	return func(_ context.Context, req racersdk.OriginRequest) (racersdk.Metadata, io.ReadCloser, error) {
+		if req.Head {
+			return metadata, nil, nil
+		}
+
+		end := int64(len(data))
+		if req.Length > 0 {
+			end = min(end, req.Offset+req.Length)
+		}
+
+		if calls.Add(1) == 1 {
+			return metadata, io.NopCloser(io.MultiReader(bytes.NewReader(data[req.Offset:cut]), racerReadError{})), nil
+		}
+
+		return metadata, io.NopCloser(bytes.NewReader(data[req.Offset:end])), nil
+	}
+}
+
+func TestRacerRetryResumesMidBody(t *testing.T) {
+	for _, offset := range []int{0, 65536} {
+		t.Run(fmt.Sprint(offset), func(t *testing.T) {
+			data := racerDistinctPayload(7, 1024*1024)
+
+			var calls atomic.Int32
+
+			client := &retryClient{Client: racerFakeClient(t, truncatingOrigin(t, data, len(data)/2+123, &calls))}
+
+			var stages retryStages
+
+			r := httptest.NewRequest(http.MethodGet, "/", nil)
+			ref := retryRef(racerDigest(data))
+			ref.Offset = int64(offset)
+
+			rec, aborted := serveRetry(retryHandler(client, 5*time.Second, &stages), r, ref)
+			if aborted || !bytes.Equal(rec.Body.Bytes(), data[offset:]) {
+				t.Fatalf("aborted=%v status=%d bytes=%d", aborted, rec.Code, rec.Body.Len())
+			}
+
+			if stages.count("write_body") < 1 || client.gets < 2 {
+				t.Fatalf("retries=%v gets=%d", stages.stages, client.gets)
+			}
+		})
+	}
+}
+
+func TestRacerRetryResumeGivesUpAfterBudget(t *testing.T) {
+	data := racerDistinctPayload(8, 1024*1024)
+	metadata := racerMetadata(t, racerDigest(data), len(data))
+	client := &retryClient{Client: racerFakeClient(t, func(_ context.Context, req racersdk.OriginRequest) (racersdk.Metadata, io.ReadCloser, error) {
+		if req.Head {
+			return metadata, nil, nil
+		}
+
+		return metadata, io.NopCloser(io.MultiReader(bytes.NewReader(data[req.Offset:req.Offset+1]), racerReadError{})), nil
+	})}
+
+	var stages retryStages
+
+	rec, aborted := serveRetry(retryHandler(client, 50*time.Millisecond, &stages), httptest.NewRequest(http.MethodGet, "/", nil), retryRef(racerDigest(data)))
+	if !aborted || rec.Body.Len() >= len(data) || !bytes.Equal(rec.Body.Bytes(), data[:rec.Body.Len()]) {
+		t.Fatalf("aborted=%v bytes=%d", aborted, rec.Body.Len())
+	}
+
+	if stages.count("write_body") < 1 {
+		t.Fatalf("retries=%v", stages.stages)
+	}
+}
+
+func TestRacerRetryResumeMetadataMismatchAborts(t *testing.T) {
+	data := racerDistinctPayload(9, 1024*1024)
+	other := racerDistinctPayload(10, 2*1024*1024)
+
+	var calls atomic.Int32
+
+	origin := truncatingOrigin(t, data, len(data)/2, &calls)
+	otherOrigin := staticOrigin(t, other)
+	fake := racerFakeClient(t, func(ctx context.Context, req racersdk.OriginRequest) (racersdk.Metadata, io.ReadCloser, error) {
+		if req.Key.String() == racerDigest(other).Hex() {
+			return otherOrigin(ctx, req)
+		}
+
+		return origin(ctx, req)
+	})
+
+	otherReq, err := Request(retryRef(racerDigest(other)), "")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	client := &retryClient{
+		Client: fake,
+		getHook: func(ctx context.Context, call int, _ racersdk.Request, opts []racersdk.ReadOptions) (*racersdk.Object, error, bool) {
+			if call == 1 {
+				return nil, nil, false
+			}
+			// Simulate a resumed read that returns a different object.
+			value, err := fake.Get(ctx, otherReq, racersdk.ReadOptions{Offset: opts[0].Offset})
+
+			return value, err, true
+		},
+	}
+
+	var stages retryStages
+
+	rec, aborted := serveRetry(retryHandler(client, 5*time.Second, &stages), httptest.NewRequest(http.MethodGet, "/", nil), retryRef(racerDigest(data)))
+	if !aborted || rec.Body.Len() >= len(data) || !bytes.Equal(rec.Body.Bytes(), data[:rec.Body.Len()]) {
+		t.Fatalf("aborted=%v bytes=%d", aborted, rec.Body.Len())
+	}
+
+	if client.gets != 2 {
+		t.Fatalf("gets=%d", client.gets)
+	}
+}
+
+func TestRacerRetryMetricStageLabels(t *testing.T) {
+	var stages retryStages
+
+	r := &retryState{policy: retryPolicy{budget: time.Second, base: time.Millisecond, cap: time.Millisecond, observe: stages.observe}, deadline: time.Now().Add(time.Second)}
+	if !r.wait(context.Background(), "get") || stages.count("get") != 1 {
+		t.Fatal("expected retry")
+	}
+
+	expired := &retryState{policy: r.policy, deadline: time.Now()}
+	if expired.wait(context.Background(), "get") {
+		t.Fatal("retried past budget")
+	}
+}
+
+// Resume through WrapHTTP over real connections, including the splice and
+// userspace fallback paths, so leftover SDK write deadlines cannot break it.
+func TestRacerRetryResumesOverHTTP(t *testing.T) {
+	for _, mode := range []string{"plaintext", "TLS", "fallback"} {
+		for _, offset := range []int{0, 65536} {
+			t.Run(fmt.Sprintf("%s/%d", mode, offset), func(t *testing.T) {
+				data := racerDistinctPayload(11, int(racersdk.PageSize)+65539)
+
+				var calls atomic.Int32
+
+				client := racerFakeClient(t, truncatingOrigin(t, data, int(racersdk.PageSize)/2+17, &calls))
+				trap := &racerDirectTrap{}
+
+				var stages retryStages
+
+				h := retryHandler(client, 5*time.Second, &stages)
+				handler := WrapHTTP(mirror.New(racerConfig(), trap, trap, mirror.WithContentBackend(h)).Handler(), time.Second, nil)
+				server, _ := handlerStreamServer(t, mode, handler)
+				rangeHeader, status := handlerRange(offset)
+
+				resp := racerRequest(t, server, http.MethodGet, "blobs", racerDigest(data), rangeHeader, "")
+				if resp.StatusCode != status {
+					t.Fatalf("status=%d", resp.StatusCode)
+				}
+
+				racerCheckStreamHash(t, resp, data, offset)
+
+				if stages.count("write_body") < 1 {
+					t.Fatalf("retries=%v", stages.stages)
+				}
+			})
+		}
+	}
+}

From 71e0e1962596d3c6f662ba1e22dac80d390e1204 Mon Sep 17 00:00:00 2001
From: Jordan Olshevski <jordan.olshevski@microsoft.com>
Date: Wed, 28 Oct 2026 18:47:37 +0000
Subject: [PATCH 08/28] gantry/racer: test retry metric stage labels

---
 internal/gantry/racer/retry_test.go | 18 +++++++++++++++++-
 1 file changed, 17 insertions(+), 1 deletion(-)

diff --git a/internal/gantry/racer/retry_test.go b/internal/gantry/racer/retry_test.go
index 7a7e26a6c..19dbf7442 100644
--- a/internal/gantry/racer/retry_test.go
+++ b/internal/gantry/racer/retry_test.go
@@ -16,8 +16,11 @@ import (
 	"testing"
 	"time"
 
+	"github.com/prometheus/client_golang/prometheus/testutil"
+
 	"github.com/Azure/unbounded/internal/gantry/digest"
 	"github.com/Azure/unbounded/internal/gantry/ifaces"
+	"github.com/Azure/unbounded/internal/gantry/metrics"
 	"github.com/Azure/unbounded/internal/gantry/mirror"
 	"github.com/Azure/unbounded/pkg/racersdk"
 )
@@ -412,7 +415,20 @@ func TestRacerRetryResumeMetadataMismatchAborts(t *testing.T) {
 	}
 }
 
-func TestRacerRetryMetricStageLabels(t *testing.T) {
+func TestRacerRetryMetrics(t *testing.T) {
+	m := NewMetrics(metrics.New())
+	m.MirrorRetry("get")
+	m.MirrorRetry("write_body")
+	m.MirrorRetry("unexpected")
+
+	for stage, want := range map[string]float64{"get": 1, "write_body": 1, "other": 1, "stat": 0} {
+		if got := testutil.ToFloat64(m.retries.WithLabelValues(stage)); got != want {
+			t.Fatalf("%s retries = %v, want %v", stage, got, want)
+		}
+	}
+}
+
+func TestRacerRetryWaitBudget(t *testing.T) {
 	var stages retryStages
 
 	r := &retryState{policy: retryPolicy{budget: time.Second, base: time.Millisecond, cap: time.Millisecond, observe: stages.observe}, deadline: time.Now().Add(time.Second)}

From 4488132440110f7fc2d878427a82c3dad96d5e98 Mon Sep 17 00:00:00 2001
From: Jordan Olshevski <jordan.olshevski@microsoft.com>
Date: Wed, 28 Oct 2026 18:47:37 +0000
Subject: [PATCH 09/28] gantry: add racer_retry_budget config and wire retry
 metrics

Expose the Racer mirror retry budget via racer_retry_budget,
GANTRY_RACER_RETRY_BUDGET, and --racer-retry-budget (default 30s, zero
disables) and report retries through gantry_racer_mirror_retries_total.
---
 cmd/gantry/agent_racer.go            |  2 +-
 internal/gantry/config/config.go     | 12 +++++++
 internal/gantry/config/racer_test.go | 54 ++++++++++++++++++++++++++++
 3 files changed, 67 insertions(+), 1 deletion(-)

diff --git a/cmd/gantry/agent_racer.go b/cmd/gantry/agent_racer.go
index 6b1a6a33f..f786744e8 100644
--- a/cmd/gantry/agent_racer.go
+++ b/cmd/gantry/agent_racer.go
@@ -85,7 +85,7 @@ func serveRacerAgent(ctx context.Context, c *config.Config, logger *slog.Logger,
 	// Always install the handler, even without a client, so unavailable requests
 	// retain Racer correlation headers and rate-limited failure diagnostics.
 	mirrorSrv := mirror.New(c, nil, originClient,
-		mirror.WithContentBackend(racer.NewHandler(deps.client, originClient, logger)), mirror.WithLogger(logger), mirror.WithStartupReadinessGate())
+		mirror.WithContentBackend(racer.NewHandler(deps.client, originClient, logger, racer.WithRetryBudget(c.RacerRetryBudget), racer.WithRetryObserver(telemetry.MirrorRetry))), mirror.WithLogger(logger), mirror.WithStartupReadinessGate())
 
 	var (
 		ready   atomic.Bool
diff --git a/internal/gantry/config/config.go b/internal/gantry/config/config.go
index f7b6b31ae..2180b0076 100644
--- a/internal/gantry/config/config.go
+++ b/internal/gantry/config/config.go
@@ -66,6 +66,11 @@ type Config struct {
 	RacerMaxConnections           int `yaml:"racer_max_connections"`
 	RacerOriginConcurrentRequests int `yaml:"racer_origin_concurrent_requests"`
 
+	// RacerRetryBudget bounds the total backoff one mirror request may spend
+	// retrying transient Racer failures, including resuming a response body
+	// after a mid-stream failure. Zero disables retries; negative is invalid.
+	RacerRetryBudget time.Duration `yaml:"racer_retry_budget"`
+
 	// ---------- Listeners ----------
 
 	// MirrorListen is the loopback address for containerd's mirror endpoint
@@ -491,6 +496,7 @@ func NewDefault() *Config {
 		RacerEnabled:                  false,
 		RacerMaxConnections:           64,
 		RacerOriginConcurrentRequests: 64,
+		RacerRetryBudget:              30 * time.Second,
 		MirrorListen:                  "127.0.0.1:5000",
 		MirrorBindAllowNonLoopback:    false,
 		TransferListen:                "0.0.0.0:5001",
@@ -690,6 +696,7 @@ func (c *Config) LoadEnv(env func(string) string) error {
 	setBool("RACER_ENABLED", &c.RacerEnabled)
 	setInt("RACER_MAX_CONNECTIONS", &c.RacerMaxConnections)
 	setInt("RACER_ORIGIN_CONCURRENT_REQUESTS", &c.RacerOriginConcurrentRequests)
+	setDur("RACER_RETRY_BUDGET", &c.RacerRetryBudget)
 	setStr("MIRROR_LISTEN", &c.MirrorListen)
 	setBool("MIRROR_BIND_ALLOW_NON_LOOPBACK", &c.MirrorBindAllowNonLoopback)
 	setStr("TRANSFER_LISTEN", &c.TransferListen)
@@ -792,6 +799,7 @@ func (c *Config) LoadEnv(env func(string) string) error {
 func (c *Config) BindFlags(fs *flag.FlagSet) {
 	fs.IntVar(&c.RacerMaxConnections, "racer-max-connections", c.RacerMaxConnections, "Racer bulk connection and live value limit (0 uses 64)")
 	fs.IntVar(&c.RacerOriginConcurrentRequests, "racer-origin-concurrent-requests", c.RacerOriginConcurrentRequests, "Racer origin concurrent GET callback and body limit (0 uses 64)")
+	fs.DurationVar(&c.RacerRetryBudget, "racer-retry-budget", c.RacerRetryBudget, "Total backoff per mirror request for retrying transient Racer failures (0 disables retries)")
 	fs.StringVar(&c.MirrorListen, "mirror-listen", c.MirrorListen, "address for the containerd-facing mirror endpoint (loopback)")
 	fs.BoolVar(&c.MirrorBindAllowNonLoopback, "mirror-bind-allow-non-loopback", c.MirrorBindAllowNonLoopback, "opt in to a non-loopback mirror bind (e.g. when using hostPort + hostIP=127.0.0.1 in Kubernetes)")
 	fs.StringVar(&c.TransferListen, "transfer-listen", c.TransferListen, "address for the peer-facing transfer endpoint")
@@ -1103,6 +1111,10 @@ func (c *Config) Validate() error {
 			}
 		}
 
+		if c.RacerRetryBudget < 0 {
+			errs = append(errs, fmt.Errorf("racer_retry_budget: must be >= 0 (zero disables retries), got %v", c.RacerRetryBudget))
+		}
+
 		return errors.Join(errs...)
 	}
 
diff --git a/internal/gantry/config/racer_test.go b/internal/gantry/config/racer_test.go
index de3258df5..2066aabe0 100644
--- a/internal/gantry/config/racer_test.go
+++ b/internal/gantry/config/racer_test.go
@@ -8,6 +8,7 @@ import (
 	"strconv"
 	"strings"
 	"testing"
+	"time"
 )
 
 func TestRacerEnabledEnvironment(t *testing.T) {
@@ -139,6 +140,59 @@ func TestRacerTuning(t *testing.T) {
 	}
 }
 
+func TestRacerRetryBudget(t *testing.T) {
+	c := NewDefault()
+	c.RacerEnabled = true
+	c.UpstreamRegistries = []UpstreamRegistry{{Name: "registry.example", Endpoint: "https://registry.example"}}
+
+	if c.RacerRetryBudget != 30*time.Second {
+		t.Fatalf("default = %v", c.RacerRetryBudget)
+	}
+
+	if err := c.LoadYAML(strings.NewReader("racer_retry_budget: 5s")); err != nil || c.RacerRetryBudget != 5*time.Second {
+		t.Fatalf("YAML = %v, err = %v", c.RacerRetryBudget, err)
+	}
+
+	env := func(value string) func(string) string {
+		return func(key string) string {
+			if key == "GANTRY_RACER_RETRY_BUDGET" {
+				return value
+			}
+
+			return ""
+		}
+	}
+
+	if err := c.LoadEnv(env("6s")); err != nil || c.RacerRetryBudget != 6*time.Second {
+		t.Fatalf("env = %v, err = %v", c.RacerRetryBudget, err)
+	}
+
+	flags := flag.NewFlagSet("test", flag.ContinueOnError)
+	c.BindFlags(flags)
+
+	if err := flags.Parse([]string{"--racer-retry-budget=7s"}); err != nil || c.RacerRetryBudget != 7*time.Second {
+		t.Fatalf("flag = %v, err = %v", c.RacerRetryBudget, err)
+	}
+
+	if err := c.Validate(); err != nil {
+		t.Fatal(err)
+	}
+
+	if err := c.LoadEnv(env("invalid")); err == nil || !strings.Contains(err.Error(), "GANTRY_RACER_RETRY_BUDGET") {
+		t.Fatalf("parse error = %v", err)
+	}
+
+	c.RacerRetryBudget = 0
+	if err := c.Validate(); err != nil {
+		t.Fatalf("zero must disable retries: %v", err)
+	}
+
+	c.RacerRetryBudget = -time.Second
+	if err := c.Validate(); err == nil || !strings.Contains(err.Error(), "racer_retry_budget") {
+		t.Fatalf("negative validation = %v", err)
+	}
+}
+
 func TestRacerEnabledEnvOnly(t *testing.T) {
 	c := NewDefault()
 	if err := c.LoadYAML(strings.NewReader("racer_enabled: true\n")); err == nil {

From f04b1b073634c0a71511e45e74af01b4e0b20537 Mon Sep 17 00:00:00 2001
From: Jordan Olshevski <jordan.olshevski@microsoft.com>
Date: Wed, 28 Oct 2026 19:00:27 +0000
Subject: [PATCH 10/28] hack/gantry-racer-bench: gantry image override, context
 guard

---
 hack/gantry-racer-bench/bench.sh             |  7 ++++++
 hack/gantry-racer-bench/gantry-override.yaml | 23 ++++++++++++++++++++
 2 files changed, 30 insertions(+)
 create mode 100644 hack/gantry-racer-bench/gantry-override.yaml

diff --git a/hack/gantry-racer-bench/bench.sh b/hack/gantry-racer-bench/bench.sh
index 8d50368aa..7959b9962 100755
--- a/hack/gantry-racer-bench/bench.sh
+++ b/hack/gantry-racer-bench/bench.sh
@@ -22,6 +22,13 @@ BUSYBOX=mcr.microsoft.com/azurelinux/busybox:1.36
 
 mkdir -p "$OUT"
 
+# Refuse to touch a cluster other than the one named in BENCH_CONTEXT. The
+# harness deletes images and writes host files on every node it can reach.
+if [[ -z ${BENCH_CONTEXT:-} || $(kubectl config current-context) != "$BENCH_CONTEXT" ]]; then
+	echo "set BENCH_CONTEXT to the current kubectl context to confirm the target cluster" >&2
+	exit 1
+fi
+
 origin_tx() {
 	kubectl -n "$NS" exec "$(tools_pod)" -- wget -qO- http://origin:5001/metrics |
 		awk '$1 == "process_network_transmit_bytes_total" {printf "%.0f\n", $2}'
diff --git a/hack/gantry-racer-bench/gantry-override.yaml b/hack/gantry-racer-bench/gantry-override.yaml
new file mode 100644
index 000000000..6537e99c7
--- /dev/null
+++ b/hack/gantry-racer-bench/gantry-override.yaml
@@ -0,0 +1,23 @@
+# Pins the gantry DaemonSet to a dev image built from this branch by the
+# images.yaml workflow, and raises the racer mirror retry budget.
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: unbounded-component-overrides
+  namespace: unbounded-system
+data:
+  gantry.yaml: |
+    apiVersion: overrides.unbounded-cloud.io/v1alpha1
+    overrides:
+      - component: gantry
+        kind: DaemonSet
+        patch:
+          spec:
+            template:
+              spec:
+                containers:
+                  - name: gantry
+                    image: ghcr.io/azure/gantry:d226e8db6efab49d5b413b80f699d41f0c863239
+                    env:
+                      - name: GANTRY_RACER_RETRY_BUDGET
+                        value: 120s

From 4609916a6cb17cd63bfbd227045b0b7630ca1f84 Mon Sep 17 00:00:00 2001
From: Jordan Olshevski <jordan.olshevski@microsoft.com>
Date: Wed, 28 Oct 2026 19:32:07 +0000
Subject: [PATCH 11/28] hack/gantry-racer-bench: per-node NIC sampling and
 stats

---
 hack/gantry-racer-bench/bench.sh    | 18 +++++++++-
 hack/gantry-racer-bench/nicstats.py | 55 +++++++++++++++++++++++++++++
 2 files changed, 72 insertions(+), 1 deletion(-)
 create mode 100644 hack/gantry-racer-bench/nicstats.py

diff --git a/hack/gantry-racer-bench/bench.sh b/hack/gantry-racer-bench/bench.sh
index 7959b9962..3505377a4 100755
--- a/hack/gantry-racer-bench/bench.sh
+++ b/hack/gantry-racer-bench/bench.sh
@@ -177,6 +177,7 @@ cmd_run() {
 	local start_at=$(($(date +%s) + LEAD))
 	local name="pull-$(date +%s)"
 	local layers=""
+	local nic_secs=${NIC_SECS:-0}
 
 	if [ "$mode" = curl ]; then
 		layers=$(kubectl -n "$NS" exec "$(tools_pod)" -- wget -qO- \
@@ -250,6 +251,16 @@ $(worker_affinity)
               done
               echo "READY \$NODE \$(date +%s.%N)"
               while [ "\$(date +%s)" -lt "\$START_AT" ]; do sleep 0.02; done
+              # Optional NIC sampler: eth0 cumulative tx:rx bytes once per second for
+              # NIC_SECS seconds, printed as one NIC line before RESULT.
+              nicf=/tmp/bench-nic.\$\$
+              if [ "$nic_secs" -gt 0 ]; then
+                (for i in \$(seq 0 $nic_secs); do
+                   echo "\$i:\$(cat /sys/class/net/eth0/statistics/tx_bytes):\$(cat /sys/class/net/eth0/statistics/rx_bytes)"
+                   sleep 1
+                 done | tr '\n' ' ' > "\$nicf") &
+                nicpid=\$!
+              fi
               t0=\$(date +%s.%N)
               rc=0
               bytes=0
@@ -266,6 +277,11 @@ $(worker_affinity)
                 rm -f "\$tmp"
               fi
               t1=\$(date +%s.%N)
+              if [ "$nic_secs" -gt 0 ]; then
+                wait "\$nicpid"
+                echo "NIC \$NODE \$(cat "\$nicf")"
+                rm -f "\$nicf"
+              fi
               echo "RESULT \$NODE \$MODE \$t0 \$t1 \$rc \$bytes"
               [ "\$rc" -eq 0 ] || echo "ERROR \$NODE \$(echo "\$out" | tr '\n' ' ' | head -c 500)"
               exec sleep infinity
@@ -316,7 +332,7 @@ EOF
 collect() {
 	local name=$1
 	kubectl -n "$NS" get pods -l run="$name" -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}' |
-		xargs -P 64 -I{} sh -c "kubectl -n $NS logs {} 2>/dev/null | grep -E '^(READY|RESULT|ERROR)' || true"
+		xargs -P 64 -I{} sh -c "kubectl -n $NS logs {} 2>/dev/null | grep -E '^(READY|RESULT|ERROR|NIC)' || true"
 }
 
 cmd_summarize() {
diff --git a/hack/gantry-racer-bench/nicstats.py b/hack/gantry-racer-bench/nicstats.py
new file mode 100644
index 000000000..9136bacf2
--- /dev/null
+++ b/hack/gantry-racer-bench/nicstats.py
@@ -0,0 +1,55 @@
+#!/usr/bin/env python3
+"""Summarize per-node NIC samples collected by `NIC_SECS=<n> bench.sh run ...`.
+
+Usage: nicstats.py <results.tsv> [step-seconds]
+
+Each NIC line is `NIC <node> 0:<tx>:<rx> 1:<tx>:<rx> ...` with cumulative eth0
+byte counters sampled once per second from the common start time. The report
+shows, per time step, the fleet distribution of per-node transmit and receive
+rates, and the distribution of each node's total transmitted bytes. Transmit
+concentrated on few nodes means a few serving nodes bound the fleet.
+"""
+
+import sys
+
+
+def pct(v, p):
+    if not v:
+        return 0.0
+    v = sorted(v)
+    return v[min(len(v) - 1, int(round(p / 100 * (len(v) - 1))))]
+
+
+def main():
+    path = sys.argv[1]
+    step = int(sys.argv[2]) if len(sys.argv) > 2 else 5
+    nodes = {}
+    for line in open(path):
+        f = line.split()
+        if not f or f[0] != "NIC":
+            continue
+        samples = []
+        for s in f[2:]:
+            i, tx, rx = s.split(":")
+            samples.append((int(i), int(tx), int(rx)))
+        nodes[f[1]] = samples
+    if not nodes:
+        sys.exit("no NIC lines")
+    n = min(len(s) for s in nodes.values())
+    mb = 1e6
+    print(f"nodes={len(nodes)} seconds={n - 1} (rates MB/s over {step}s windows)")
+    print(f"{'t':>4} {'tx p50':>8} {'tx p90':>8} {'tx p99':>8} {'tx max':>8} {'rx p50':>8} {'rx p99':>8} {'fleet rx GB/s':>14}")
+    for t in range(step, n, step):
+        tx = [(s[t][1] - s[t - step][1]) / step / mb for s in nodes.values()]
+        rx = [(s[t][2] - s[t - step][2]) / step / mb for s in nodes.values()]
+        print(f"{t:>4} {pct(tx, 50):8.0f} {pct(tx, 90):8.0f} {pct(tx, 99):8.0f} {max(tx):8.0f} "
+              f"{pct(rx, 50):8.0f} {pct(rx, 99):8.0f} {sum(rx) / 1000:14.1f}")
+    tot = [(s[n - 1][1] - s[0][1]) / 1e9 for s in nodes.values()]
+    peak = [max((s[t][1] - s[t - 1][1]) / mb for t in range(1, n)) for s in nodes.values()]
+    print(f"per-node total tx GB: p10={pct(tot, 10):.2f} p50={pct(tot, 50):.2f} p90={pct(tot, 90):.2f} "
+          f"p99={pct(tot, 99):.2f} max={max(tot):.2f} sum={sum(tot):.0f}")
+    print(f"per-node peak 1s tx MB/s: p50={pct(peak, 50):.0f} p90={pct(peak, 90):.0f} p99={pct(peak, 99):.0f} max={max(peak):.0f}")
+
+
+if __name__ == "__main__":
+    main()

From 81e482da7a38c95cdb7f13a71314a05de0adf397 Mon Sep 17 00:00:00 2001
From: Jordan Olshevski <jordan.olshevski@microsoft.com>
Date: Wed, 28 Oct 2026 20:06:17 +0000
Subject: [PATCH 12/28] hack/gantry-racer-bench: replicated origin, safe log
 collection

---
 hack/gantry-racer-bench/bench.sh         | 41 +++++++++++++++++++++---
 hack/gantry-racer-bench/genimage/main.go |  4 ++-
 hack/gantry-racer-bench/nicstats.py      | 13 ++++++--
 hack/gantry-racer-bench/origin.yaml      | 21 +++++++++---
 4 files changed, 66 insertions(+), 13 deletions(-)

diff --git a/hack/gantry-racer-bench/bench.sh b/hack/gantry-racer-bench/bench.sh
index 3505377a4..b53686fd6 100755
--- a/hack/gantry-racer-bench/bench.sh
+++ b/hack/gantry-racer-bench/bench.sh
@@ -29,9 +29,19 @@ if [[ -z ${BENCH_CONTEXT:-} || $(kubectl config current-context) != "$BENCH_CONT
 	exit 1
 fi
 
+# origin_ips lists the IPs of every origin registry replica. The Service
+# spreads connections over them, so each replica must hold the same images.
+origin_ips() {
+	kubectl -n "$NS" get pod -l app=origin -o jsonpath='{range .items[*]}{.status.podIP}{"\n"}{end}'
+}
+
+# origin_tx sums network egress over all origin replicas.
 origin_tx() {
-	kubectl -n "$NS" exec "$(tools_pod)" -- wget -qO- http://origin:5001/metrics |
-		awk '$1 == "process_network_transmit_bytes_total" {printf "%.0f\n", $2}'
+	local tools ip
+	tools=$(tools_pod)
+	for ip in $(origin_ips); do
+		kubectl -n "$NS" exec "$tools" -- wget -qO- "http://$ip:5001/metrics"
+	done | awk '$1 == "process_network_transmit_bytes_total" {s += $2} END {printf "%.0f\n", s}'
 }
 
 tools_pod() {
@@ -165,10 +175,25 @@ EOF
 	kubectl -n "$NS" delete ds diskprep --wait=false >/dev/null
 }
 
+# cmd_gen pushes the same image (fixed seed and creation time) to every
+# origin replica and prints its repo@digest once all pushes agree.
 cmd_gen() {
 	local name=$1 layers=$2 bytes=$3
-	kubectl -n "$NS" exec "$(tools_pod)" -- /tmp/genimage -registry http://origin:5000 \
-		-repo "bench/$name" -layers "$layers" -layer-bytes "$bytes" -parallel 8
+	local tools seed created ip refs
+	tools=$(tools_pod)
+	seed=$(od -An -N8 -tu8 /dev/urandom | tr -d ' ')
+	created=$(date -u +%Y-%m-%dT%H:%M:%SZ)
+	refs=$(for ip in $(origin_ips); do
+		kubectl -n "$NS" exec "$tools" -- /tmp/genimage -registry "http://$ip:5000" \
+			-repo "bench/$name" -layers "$layers" -layer-bytes "$bytes" -parallel 8 \
+			-seed "$seed" -created "$created" &
+	done | sort -u)
+	wait
+	if [ "$(printf '%s\n' "$refs" | grep -c .)" -ne 1 ]; then
+		echo "origin replicas disagree: $refs" >&2
+		exit 1
+	fi
+	echo "$refs"
 }
 
 cmd_run() {
@@ -331,8 +356,14 @@ EOF
 # collect prints READY/RESULT/ERROR lines from every pod of a run.
 collect() {
 	local name=$1
+	# Each pod's lines go to their own file so that concurrent writers cannot
+	# interleave long NIC lines (pipe writes over PIPE_BUF are not atomic).
+	local dir
+	dir=$(mktemp -d "$OUT/collect.XXXXXX")
 	kubectl -n "$NS" get pods -l run="$name" -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}' |
-		xargs -P 64 -I{} sh -c "kubectl -n $NS logs {} 2>/dev/null | grep -E '^(READY|RESULT|ERROR|NIC)' || true"
+		xargs -P 64 -I{} sh -c "kubectl -n $NS logs {} 2>/dev/null | grep -E '^(READY|RESULT|ERROR|NIC)' > $dir/{} || true"
+	cat "$dir"/* 2>/dev/null || true
+	rm -rf "$dir"
 }
 
 cmd_summarize() {
diff --git a/hack/gantry-racer-bench/genimage/main.go b/hack/gantry-racer-bench/genimage/main.go
index f2d2f7fae..cfd111d39 100644
--- a/hack/gantry-racer-bench/genimage/main.go
+++ b/hack/gantry-racer-bench/genimage/main.go
@@ -46,6 +46,8 @@ func main() {
 	layerBytes := flag.Int64("layer-bytes", 1<<30, "random payload bytes per layer")
 	seed := flag.Uint64("seed", uint64(time.Now().UnixNano()), "PRNG seed")
 	parallel := flag.Int("parallel", 8, "layers generated and pushed in parallel")
+	created := flag.String("created", time.Now().UTC().Format(time.RFC3339),
+		"image config creation time; fix it together with -seed to push identical images to several registries")
 
 	flag.Parse()
 
@@ -88,7 +90,7 @@ func main() {
 	config := map[string]any{
 		"architecture": "amd64",
 		"os":           "linux",
-		"created":      time.Now().UTC().Format(time.RFC3339),
+		"created":      *created,
 		"config":       map[string]any{"Cmd": []string{"/data/none"}},
 		"rootfs":       map[string]any{"type": "layers", "diff_ids": diffIDs},
 	}
diff --git a/hack/gantry-racer-bench/nicstats.py b/hack/gantry-racer-bench/nicstats.py
index 9136bacf2..01349f9b4 100644
--- a/hack/gantry-racer-bench/nicstats.py
+++ b/hack/gantry-racer-bench/nicstats.py
@@ -24,17 +24,24 @@ def main():
     path = sys.argv[1]
     step = int(sys.argv[2]) if len(sys.argv) > 2 else 5
     nodes = {}
+    skipped = 0
     for line in open(path):
         f = line.split()
         if not f or f[0] != "NIC":
             continue
         samples = []
-        for s in f[2:]:
-            i, tx, rx = s.split(":")
-            samples.append((int(i), int(tx), int(rx)))
+        try:
+            for s in f[2:]:
+                i, tx, rx = s.split(":")
+                samples.append((int(i), int(tx), int(rx)))
+        except ValueError:
+            skipped += 1
+            continue
         nodes[f[1]] = samples
     if not nodes:
         sys.exit("no NIC lines")
+    if skipped:
+        print(f"skipped {skipped} malformed NIC lines")
     n = min(len(s) for s in nodes.values())
     mb = 1e6
     print(f"nodes={len(nodes)} seconds={n - 1} (rates MB/s over {step}s windows)")
diff --git a/hack/gantry-racer-bench/origin.yaml b/hack/gantry-racer-bench/origin.yaml
index 473941ee7..6cd67576b 100644
--- a/hack/gantry-racer-bench/origin.yaml
+++ b/hack/gantry-racer-bench/origin.yaml
@@ -1,6 +1,8 @@
 # In-cluster origin registry for the Gantry vs Gantry+Racer benchmark.
-# Plain HTTP distribution registry pinned to one system node, backed by the
-# node's local disk, so origin bandwidth is the same for every run.
+# Plain HTTP distribution registry, one replica per system node, each backed by
+# its node's local disk. Replicas do not share storage: `bench.sh gen` pushes
+# identical images to every replica, and the Service spreads pulls over them so
+# origin bandwidth is the same for every run.
 apiVersion: v1
 kind: Namespace
 metadata:
@@ -12,7 +14,11 @@ metadata:
   name: origin
   namespace: gantry-bench
 spec:
-  replicas: 1
+  replicas: 3
+  # Replicas are pinned one per system node by anti-affinity, so a rolling
+  # update has nowhere to place a surge pod.
+  strategy:
+    type: Recreate
   selector:
     matchLabels:
       app: origin
@@ -22,7 +28,14 @@ spec:
         app: origin
     spec:
       nodeSelector:
-        kubernetes.io/hostname: aks-system-36530849-vmss000002
+        kubernetes.azure.com/mode: system
+      affinity:
+        podAntiAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            - labelSelector:
+                matchLabels:
+                  app: origin
+              topologyKey: kubernetes.io/hostname
       containers:
         - name: registry
           image: ghcr.io/distribution/distribution:3.0.0

From ddc3b4cc583ff929eea2e587bce8b8c72eafb93f Mon Sep 17 00:00:00 2001
From: Jordan Olshevski <jordan.olshevski@microsoft.com>
Date: Wed, 28 Oct 2026 20:45:45 +0000
Subject: [PATCH 13/28] hack/gantry-racer-bench: results

---
 hack/gantry-racer-bench/RESULTS.md | 152 +++++++++++++++++++++++++++++
 1 file changed, 152 insertions(+)
 create mode 100644 hack/gantry-racer-bench/RESULTS.md

diff --git a/hack/gantry-racer-bench/RESULTS.md b/hack/gantry-racer-bench/RESULTS.md
new file mode 100644
index 000000000..b4578f884
--- /dev/null
+++ b/hack/gantry-racer-bench/RESULTS.md
@@ -0,0 +1,152 @@
+# Gantry vs Gantry + Racer: image pull benchmark
+
+Date: 2026-10-28. Base: origin/main d017881b5, plus the fixes listed below.
+
+## Setup
+
+- AKS cluster `joolshev-scale-test`, 1497 worker nodes. Every node is 8 vCPU / 32 GiB.
+  - 447 Standard_D8ads_v5 and 550 Standard_D8d_v5. Racer uses the 43 GiB local temp disk as a raw block device.
+  - 500 Standard_D8ds_v6. These have no extra local disk, so Racer uses 6 GiB slab files per I/O worker on the
+    OS disk.
+- Origin: `distribution` 3.0.0 over plain HTTP on the 3 system nodes, one replica per node. Every image is pushed
+  to all 3 replicas (`origin.yaml`, `bench.sh gen`).
+- Images are random data, so they don't compress or dedupe:
+  - 8 GiB: 8 layers of 1 GiB.
+  - 1 GiB: 4 layers of 256 MiB.
+  - Smoke: 2 layers of 64 MiB.
+  - Every run uses a new image.
+- Each run starts a DaemonSet that pulls with `crictl pull` on every worker at the same second. The time reported
+  is from that start until the pull finishes, so it includes the containerd unpack.
+- containerd uses gantry first, then falls back to the origin directly (`certs.d/bench.registry.local/hosts.toml`).
+- Racer runs use one dataplane per node. RDMA is off. Racer settings are in "Racer config" below.
+
+## Results (3 origin replicas)
+
+| Image | Mode | Run | p50 | p90 | p95 | p99 | max | Origin egress |
+|---|---|---|---|---|---|---|---|---|
+| 8 GiB | Gantry P2P | o3a | 75.9s | 81.2s | 82.6s | 85.0s | 96.3s | 64.1 GiB |
+| 8 GiB | Gantry P2P | o3b | 76.2s | 80.7s | 81.9s | 84.8s | 95.7s | 64.1 GiB |
+| 8 GiB | Gantry + Racer | o3c | 54.1s | 61.6s | 63.4s | 66.2s | 72.1s | 12.5 GiB |
+| 8 GiB | Gantry + Racer | o3d | 52.2s | 58.8s | 60.8s | 64.9s | 73.8s | 12.9 GiB |
+| 1 GiB | Gantry P2P | sa | 15.5s | 16.2s | 16.6s | 17.5s | 18.4s | 8.0 GiB |
+| 1 GiB | Gantry P2P | sb | 15.2s | 16.3s | 16.7s | 17.4s | 17.8s | 8.0 GiB |
+| 1 GiB | Gantry + Racer | sc | 14.6s | 16.7s | 17.1s | 17.7s | 18.7s | 1.6 GiB |
+| 1 GiB | Gantry + Racer | sd | 15.8s | 17.7s | 18.1s | 19.2s | 20.3s | 1.6 GiB |
+| 128 MiB | Gantry + Racer | smoke | 8.6s | 9.7s | 9.9s | 10.0s | 11.0s | 0.2 GiB |
+
+All runs: 1497 of 1497 nodes succeeded. The results were close across node pools. In the 8 GiB Racer runs the
+D8ads_v5 pool was 3-5s slower than the other two at p50.
+
+### Same tests with a single origin replica (8 GiB)
+
+| Mode | Runs | p50 | p95 | max | Origin egress |
+|---|---|---|---|---|---|
+| Gantry P2P | k, l | 110.2s, 94.8s | 116.7s, 100.2s | 123.4s, 108.4s | 64.2 GiB |
+| Gantry + Racer | e, h, j | 54.6s, 56.4s, 55.7s | 62.8s, 65.2s, 64.3s | 74.5s, 75.0s, 77.4s | 12.3-12.5 GiB |
+
+Gantry P2P depends on origin bandwidth. It fetches each layer from the origin about 8 times. On one origin node
+(about 1.6 GB/s NIC), the first 35s of the gantry run moved only about 2 GB/s across the whole fleet. Racer
+fetches each 16 MiB page from the origin about 1.5 times, so origin bandwidth barely affects it.
+
+### Fetch only (Racer, 8 GiB, curl from the gantry mirror, no unpack)
+
+- 3 concurrent blobs per node: p50 45.9s, max 59.7s.
+- 8 concurrent blobs per node: p50 49.0s, max 55.0s.
+
+Unpack adds about 6-8s. More parallel streams per node don't help.
+
+### Network (8 GiB, per-node NIC counters sampled every second)
+
+| | Gantry P2P (o3a) | Gantry + Racer (o3c, o3d) |
+|---|---|---|
+| Total bytes sent by all nodes | 13.9 TB (1.08x the bytes delivered) | 27.1 TB (2.1x) |
+| Bytes sent per node, p50 / p99 / max | 7.0 / 36.1 / 50.1 GB | 17.3 / 45.4 / 56.6 GB |
+| Peak fleet receive rate | 479 GB/s | 718-732 GB/s |
+| Time until network goes quiet | ~70s | ~50s |
+
+## What limits each mode
+
+**Gantry P2P**
+- Startup is slow. All nodes pull the layers in the same order, and the first copies come from the origin. With 3
+  origin replicas the fleet receive rate was 4 GB/s at 10s, 70 GB/s at 30s, and 480 GB/s at 50s.
+- Once peers have data, the fleet peaks at about 480 GB/s.
+
+**Gantry + Racer**
+- Each 16 MiB page has 3 owner nodes (`cmd/racer-dataplane/topology/src/placement.rs:10`, a protocol constant).
+  Only owners serve a page to other nodes. A node that already holds the page does not serve it.
+- With 512 pages and 1497 nodes, some nodes own several pages. Each owned page means about 8 GB of upload, so
+  these nodes run their NIC at line rate (about 1.6 GB/s) for most of the run and set the tail.
+- Each node connects directly to at most 64 neighbors (`topology/src/lib.rs`, 32 rings). It reaches other owners
+  through a relay node, so every byte crosses the network about 2.1 times.
+- Fixing these needs design changes (non-owners serving pages, or fan-out through relays). No config knob changes
+  them.
+
+## Changes made (draft PR, branch `bench/gantry-racer`)
+
+1. **Gantry: retry and resume Racer reads** (`internal/gantry/racer/racer.go`).
+   - Before: when an owner is busy, Racer returns unavailable. Gantry turned that into a 503, or cut off a blob
+     that was already streaming. containerd then fetched from the origin. With 8 GiB images and the default Racer
+     config, only 34 of 1497 nodes finished in 15 minutes and the origin ran at full speed. The 128 MiB smoke test
+     took 22.5s p50 and pulled 18 GiB from the origin.
+   - Now gantry retries unavailable errors with backoff (150ms doubling to 2s, with jitter). It resumes a broken
+     blob stream at the byte where it stopped, pinned to the same ETag.
+   - New setting `racer_retry_budget` (`GANTRY_RACER_RETRY_BUDGET`, default 30s, 0 turns it off). The benchmark
+     used 120s.
+   - New metric `gantry_racer_mirror_retries_total{stage}`.
+2. **Racer: recover when the slab size changes** (`cmd/racer-dataplane/alloc/src/slab.rs`, `src/app.rs`).
+   - Before: changing `RACER_SLAB_BYTES` made every slab-file node crash on start (`InvalidConfiguration`) until
+     someone deleted the old files by hand.
+   - Now slab files of the wrong size are recreated empty. Files of workers that no longer exist are deleted.
+     Raw device mode is not changed.
+3. **Benchmark tools** in this directory.
+
+## Racer config
+
+These values are set in ConfigMap `unbounded-system/racer-dataplane-config`. Byte and count values are
+node-wide.
+
+| Key | Value | Default | Why |
+|---|---|---|---|
+| `RACER_ALLOW_SMT` | true | false | Use all 8 vCPUs (5 I/O + 3 crypto threads) instead of 4 physical cores (3 + 1). |
+| `RACER_ENABLE_RDMA` | false | auto | No RDMA on these VMs. |
+| `RACER_SLAB_BYTES` | 6 GiB | 1 GiB | Larger cache on nodes without a local disk. |
+| `RACER_CIPHERTEXT_BYTES` | 8 GiB | 128 MiB x workers | The default budget filled up and rejected peer requests. |
+| `RACER_PLAINTEXT_BYTES` | 2 GiB | 64 MiB x workers | Same. |
+| `RACER_DIRTY_BYTES` | 2 GiB | 160 MiB x workers | Same. |
+| `RACER_REQUEST_CONTEXT_BYTES` | 512 MiB | 32 MiB x workers | Same. |
+| `RACER_CLIENT_CONNECTIONS` | 2048 | 64 x workers | Outbound connection limit (15 per worker) was hit. |
+| `RACER_PIPES` | 1024 | 16 x workers | More concurrent peer transfers. |
+| `RACER_QUEUE_ENTRIES` | 4096 | 256 x workers | Same. |
+| `RACER_WAITERS_PER_FLIGHT` | 2048 | 64 | Up to 1497 readers wait on the same page. |
+| `RACER_PEER_INFLIGHT_MAX` | 1024 | 256 | More outbound requests. |
+| `RACER_PEER_PER_NEIGHBOR_MAX` | 128 | 32 | Relays forward many requests to one owner. |
+| `RACER_CONNECTIONS_PER_NEIGHBOR` | 4 | 2 | Less head-of-line blocking on 16 MiB pages. |
+| `RACER_RELAY_TRANSFERS` | 256 | 16 x workers | More relayed transfers. |
+
+D8ads_v5 and D8d_v5 nodes: `diskprep.sh` unmounts `/mnt` and wipes `/dev/sdb`. The node is then annotated with
+`racer.unbounded-cloud.io/block-devices=^wwn-0x...$`.
+
+## Other findings
+
+- Gantry P2P needs a direct path from containerd to the registry. Its mirror returns 503 on purpose when no peer
+  has a layer yet (`internal/gantry/mirror/mirror.go`). Without the fallback, 18 of 1497 nodes failed.
+- Changing a Racer setting rolls the DaemonSet with `maxUnavailable: 1`, which takes more than 15 minutes on 1500
+  nodes. The benchmark deleted the pods instead.
+- During a full restart, 2 of 1500 Racer pods exited once with `fatal stage=run error=Unauthorized` and then
+  recovered.
+- Racer does not log request failures. Use `/debug/failures` on the metrics port (9090).
+
+## Running it
+
+```sh
+export BENCH_CONTEXT=$(kubectl config current-context)
+kubectl apply -f origin.yaml
+./bench.sh prep                      # hosts.toml on every worker
+./bench.sh diskprep                  # optional: free local disks for Racer
+ref=$(./bench.sh gen g8x1g-a 8 1073741824)
+NIC_SECS=100 ./bench.sh run gantry-a "$ref" crictl
+python3 nicstats.py ../../tmp/gantry-racer-bench/gantry-a.tsv 10
+```
+
+Racer mode is on when a ClusterCache named `gantry` exists. Delete it to go back to gantry P2P. To keep the Racer
+DaemonSet while doing that, first create another ClusterCache, for example `bench-hold`.

From 141224aa3a12a229bda1bdbd23c15ce76e4dc16a Mon Sep 17 00:00:00 2001
From: Jordan Olshevski <jordan.olshevski@microsoft.com>
Date: Wed, 28 Oct 2026 22:30:39 +0000
Subject: [PATCH 14/28] hack/gantry-racer-bench: prune leased content,
 interrupt-safe collection

---
 hack/gantry-racer-bench/bench.sh | 81 +++++++++++++++++++++++++++++++-
 1 file changed, 80 insertions(+), 1 deletion(-)

diff --git a/hack/gantry-racer-bench/bench.sh b/hack/gantry-racer-bench/bench.sh
index b53686fd6..8768f2d63 100755
--- a/hack/gantry-racer-bench/bench.sh
+++ b/hack/gantry-racer-bench/bench.sh
@@ -10,6 +10,7 @@
 #                        mode=crictl: full containerd pull + unpack
 #                        mode=curl:   fetch every layer blob through the local gantry mirror
 #   summarize <results.tsv>
+#   prune                remove unused images (incl. old bench images) on every worker
 set -euo pipefail
 
 NS=gantry-bench
@@ -120,6 +121,74 @@ EOF
 	kubectl -n "$NS" rollout status ds/prep --timeout=15m
 }
 
+# cmd_prune removes every image that no container uses on every worker, which
+# includes earlier bench images, and prints the 5 fullest root disks. It also
+# drops gantry's content leases. The 40 GB runs need this: content plus
+# snapshot is about 80 GB on a 123 GiB OS disk, and blobs still leased from the
+# previous run can push a node past the kubelet eviction limit.
+cmd_prune() {
+	kubectl apply -f - <<EOF
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: prune
+  namespace: $NS
+spec:
+  selector:
+    matchLabels:
+      app: prune
+  template:
+    metadata:
+      labels:
+        app: prune
+    spec:
+      terminationGracePeriodSeconds: 1
+$(worker_affinity)
+      tolerations:
+        - operator: Exists
+      containers:
+        - name: prune
+          image: $BUSYBOX
+          securityContext:
+            privileged: true
+          env:
+            - name: NODE
+              valueFrom:
+                fieldRef:
+                  fieldPath: spec.nodeName
+          command:
+            - chroot
+            - /host
+            - bash
+            - -c
+            - |
+              # Gantry P2P leases the blobs it ingests for 60 minutes
+              # (containerd_lease_ttl), which keeps them on disk after the
+              # image is removed. Drop those leases first.
+              ctr -n k8s.io leases ls -q | grep '^gantry-' |
+                xargs -r ctr -n k8s.io leases rm --sync >/dev/null 2>&1 || true
+              crictl rmi --prune >/dev/null 2>&1 || true
+              ctr -n k8s.io content prune references >/dev/null 2>&1 || true
+              echo "PRUNED \$NODE \$(df --output=pcent / | tail -n 1 | tr -d ' %')"
+              exec sleep infinity
+          resources:
+            requests:
+              cpu: 10m
+              memory: 16Mi
+          volumeMounts:
+            - name: host
+              mountPath: /host
+      volumes:
+        - name: host
+          hostPath:
+            path: /
+EOF
+	kubectl -n "$NS" rollout status ds/prune --timeout=15m
+	kubectl -n "$NS" get pods -l app=prune -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}' |
+		xargs -P 64 -I{} kubectl -n "$NS" logs {} 2>/dev/null | grep '^PRUNED' | sort -k3 -n | tail -n 5
+	kubectl -n "$NS" delete ds prune --wait=false >/dev/null
+}
+
 # cmd_diskprep frees the Azure resource disk (mounted at /mnt) on every worker
 # that has one, zeroes its head and tail so racer's device guard accepts it,
 # and annotates the Node so racer uses it as a raw block device.
@@ -330,13 +399,22 @@ EOF
 	# Wait for every pod to report a result.
 	while true; do
 		sleep 20
-		collect "$name" >"$results.tmp"
+		# Write to a scratch file first so an interrupted collect does not wipe
+		# the last good snapshot.
+		collect "$name" >"$results.tmp.new" && mv "$results.tmp.new" "$results.tmp"
 		local got
 		got=$(grep -c '^RESULT' "$results.tmp" || true)
 		echo "$(date -u +%T) results $got/$want" >&2
 		if [ "$got" -ge "$want" ]; then
 			break
 		fi
+		# Touch $OUT/<label>.stop to stop waiting and keep the results so far,
+		# for example when some nodes were evicted and will never report.
+		if [ -e "$OUT/$label.stop" ]; then
+			echo "stop file found" >&2
+			rm -f "$OUT/$label.stop"
+			break
+		fi
 		if [ "$(date +%s)" -gt $((start_at + ${MAX_WAIT:-3600})) ]; then
 			echo "timed out waiting for results" >&2
 			break
@@ -410,5 +488,6 @@ diskprep) cmd_diskprep ;;
 gen) shift; cmd_gen "$@" ;;
 run) shift; cmd_run "$@" ;;
 summarize) shift; cmd_summarize "$@" ;;
+prune) cmd_prune ;;
 *) sed -n '2,13p' "$0"; exit 1 ;;
 esac

From 88f44da0e8b7061c970c1d8575c2b0e335c2a0c6 Mon Sep 17 00:00:00 2001
From: Jordan Olshevski <jordan.olshevski@microsoft.com>
Date: Wed, 28 Oct 2026 22:32:37 +0000
Subject: [PATCH 15/28] hack/gantry-racer-bench: add 40 GB results

---
 hack/gantry-racer-bench/RESULTS.md | 155 ++++++++++++++++++-----------
 1 file changed, 97 insertions(+), 58 deletions(-)

diff --git a/hack/gantry-racer-bench/RESULTS.md b/hack/gantry-racer-bench/RESULTS.md
index b4578f884..0dd76d748 100644
--- a/hack/gantry-racer-bench/RESULTS.md
+++ b/hack/gantry-racer-bench/RESULTS.md
@@ -1,41 +1,64 @@
 # Gantry vs Gantry + Racer: image pull benchmark
 
-Date: 2026-10-28. Base: origin/main d017881b5, plus the fixes listed below.
+Date: 2026-10-28. Base: origin/main d017881b5, plus the fixes listed under "Changes made" (draft PR
+https://github.com/Azure/unbounded/pull/1016).
 
 ## Setup
 
-- AKS cluster `joolshev-scale-test`, 1497 worker nodes. Every node is 8 vCPU / 32 GiB.
+- AKS cluster `joolshev-scale-test`, 1497 worker nodes. Every node is 8 vCPU / 32 GiB with a 128 GB local OS disk.
   - 447 Standard_D8ads_v5 and 550 Standard_D8d_v5. Racer uses the 43 GiB local temp disk as a raw block device.
   - 500 Standard_D8ds_v6. These have no extra local disk, so Racer uses 6 GiB slab files per I/O worker on the
     OS disk.
 - Origin: `distribution` 3.0.0 over plain HTTP on the 3 system nodes, one replica per node. Every image is pushed
-  to all 3 replicas (`origin.yaml`, `bench.sh gen`).
-- Images are random data, so they don't compress or dedupe:
+  to all 3 replicas. (Some earlier runs used 1 replica; they are listed separately.)
+- Images are random data, so they don't compress or dedupe. Every run uses a new image.
+  - 40 GB: 40 layers of 1 GB (10^9 bytes; 37.3 GiB total).
   - 8 GiB: 8 layers of 1 GiB.
   - 1 GiB: 4 layers of 256 MiB.
   - Smoke: 2 layers of 64 MiB.
-  - Every run uses a new image.
-- Each run starts a DaemonSet that pulls with `crictl pull` on every worker at the same second. The time reported
-  is from that start until the pull finishes, so it includes the containerd unpack.
+- Each run starts a DaemonSet that runs `crictl pull` on every worker at the same second (start skew under
+  0.05s). The time reported is from that start until the pull finishes, so it includes the containerd unpack.
 - containerd uses gantry first, then falls back to the origin directly (`certs.d/bench.registry.local/hosts.toml`).
-- Racer runs use one dataplane per node. RDMA is off. Racer settings are in "Racer config" below.
-
-## Results (3 origin replicas)
+- Same gantry image in both modes (main + retry/resume fix). Racer mode is on when ClusterCache `gantry` exists.
+- Racer: one dataplane per node, RDMA off, tuned config (see "Racer config").
+
+## Full results (3 origin replicas, 1497 nodes, all runs 1497/1497 succeeded)
+
+| Image | Mode | Run | p50 | p90 | p95 | p99 | max | mean | Origin egress |
+|---|---|---|---|---|---|---|---|---|---|
+| 40 GB | Gantry P2P | 40g-c | 279.9s | 320.4s | 333.3s | 362.1s | 408.0s | 281.6s | 299.2 GiB |
+| 40 GB | Gantry P2P | 40g-e | 280.0s | 327.2s | 343.6s | 369.6s | 404.9s | 284.3s | 299.2 GiB |
+| 40 GB | Gantry + Racer | 40g-a | 233.3s | 278.3s | 296.1s | 323.3s | 352.8s | 234.6s | 61.7 GiB |
+| 40 GB | Gantry + Racer | 40g-b | 228.2s | 273.7s | 290.3s | 312.6s | 328.6s | 230.9s | 62.5 GiB |
+| 8 GiB | Gantry P2P | o3a | 75.9s | 81.2s | 82.6s | 85.0s | 96.3s | | 64.1 GiB |
+| 8 GiB | Gantry P2P | o3b | 76.2s | 80.7s | 81.9s | 84.8s | 95.7s | | 64.1 GiB |
+| 8 GiB | Gantry + Racer | o3c | 54.1s | 61.6s | 63.4s | 66.2s | 72.1s | | 12.5 GiB |
+| 8 GiB | Gantry + Racer | o3d | 52.2s | 58.8s | 60.8s | 64.9s | 73.8s | | 12.9 GiB |
+| 1 GiB | Gantry P2P | sa | 15.5s | 16.2s | 16.6s | 17.5s | 18.4s | | 8.0 GiB |
+| 1 GiB | Gantry P2P | sb | 15.2s | 16.3s | 16.7s | 17.4s | 17.8s | | 8.0 GiB |
+| 1 GiB | Gantry + Racer | sc | 14.6s | 16.7s | 17.1s | 17.7s | 18.7s | | 1.6 GiB |
+| 1 GiB | Gantry + Racer | sd | 15.8s | 17.7s | 18.1s | 19.2s | 20.3s | | 1.6 GiB |
+| 128 MiB | Gantry + Racer | smoke | 8.6s | 9.7s | 9.9s | 10.0s | 11.0s | | 0.2 GiB |
+
+### Summary
+
+| Image | Racer vs Gantry P2P, p50 | p99 | max | Origin egress |
+|---|---|---|---|---|
+| 40 GB | 18% faster (231s vs 280s) | 13% faster | 16% faster | 4.8x less (1.7x image vs 8.0x image) |
+| 8 GiB | 30% faster (53s vs 76s) | 22% faster | 24% faster | 5x less (1.6x image vs 8.0x image) |
+| 1 GiB | same (15s) | 5-10% slower | 5-10% slower | 5x less |
+
+### 40 GB results by node pool (p50 / p90 / max)
+
+| Run | D8ads_v5 (raw disk) | D8ds_v6 (slab on OS disk) | D8d_v5 (raw disk) |
+|---|---|---|---|
+| Gantry 40g-c | 282.6 / 302.9 / 342.5s | 265.5 / 328.9 / 368.6s | 284.7 / 325.3 / 408.0s |
+| Gantry 40g-e | 282.5 / 303.6 / 336.6s | 269.3 / 332.2 / 380.3s | 284.9 / 336.5 / 404.9s |
+| Racer 40g-a | 239.2 / 257.4 / 283.8s | 217.0 / 293.1 / 352.8s | 233.7 / 278.3 / 335.7s |
+| Racer 40g-b | 233.6 / 250.3 / 268.6s | 208.2 / 285.5 / 325.9s | 227.0 / 276.3 / 328.6s |
 
-| Image | Mode | Run | p50 | p90 | p95 | p99 | max | Origin egress |
-|---|---|---|---|---|---|---|---|---|
-| 8 GiB | Gantry P2P | o3a | 75.9s | 81.2s | 82.6s | 85.0s | 96.3s | 64.1 GiB |
-| 8 GiB | Gantry P2P | o3b | 76.2s | 80.7s | 81.9s | 84.8s | 95.7s | 64.1 GiB |
-| 8 GiB | Gantry + Racer | o3c | 54.1s | 61.6s | 63.4s | 66.2s | 72.1s | 12.5 GiB |
-| 8 GiB | Gantry + Racer | o3d | 52.2s | 58.8s | 60.8s | 64.9s | 73.8s | 12.9 GiB |
-| 1 GiB | Gantry P2P | sa | 15.5s | 16.2s | 16.6s | 17.5s | 18.4s | 8.0 GiB |
-| 1 GiB | Gantry P2P | sb | 15.2s | 16.3s | 16.7s | 17.4s | 17.8s | 8.0 GiB |
-| 1 GiB | Gantry + Racer | sc | 14.6s | 16.7s | 17.1s | 17.7s | 18.7s | 1.6 GiB |
-| 1 GiB | Gantry + Racer | sd | 15.8s | 17.7s | 18.1s | 19.2s | 20.3s | 1.6 GiB |
-| 128 MiB | Gantry + Racer | smoke | 8.6s | 9.7s | 9.9s | 10.0s | 11.0s | 0.2 GiB |
-
-All runs: 1497 of 1497 nodes succeeded. The results were close across node pools. In the 8 GiB Racer runs the
-D8ads_v5 pool was 3-5s slower than the other two at p50.
+D8ds_v6 nodes have the best p50 but the widest spread in both modes. (Inference: they have a faster NVMe OS disk,
+but in Racer mode the slab cache shares that disk with containerd.)
 
 ### Same tests with a single origin replica (8 GiB)
 
@@ -46,42 +69,58 @@ D8ads_v5 pool was 3-5s slower than the other two at p50.
 
 Gantry P2P depends on origin bandwidth. It fetches each layer from the origin about 8 times. On one origin node
 (about 1.6 GB/s NIC), the first 35s of the gantry run moved only about 2 GB/s across the whole fleet. Racer
-fetches each 16 MiB page from the origin about 1.5 times, so origin bandwidth barely affects it.
+fetches each 16 MiB page from the origin about 1.5-1.7 times, so origin bandwidth barely affects it.
 
-### Fetch only (Racer, 8 GiB, curl from the gantry mirror, no unpack)
+### Fetch only (Racer, 8 GiB, curl from the gantry mirror, no unpack, 1 origin)
 
 - 3 concurrent blobs per node: p50 45.9s, max 59.7s.
 - 8 concurrent blobs per node: p50 49.0s, max 55.0s.
 
-Unpack adds about 6-8s. More parallel streams per node don't help.
+Unpack adds about 6-8s for 8 GiB. More parallel streams per node don't help.
 
-### Network (8 GiB, per-node NIC counters sampled every second)
+## Network (per-node NIC counters sampled every second)
 
-| | Gantry P2P (o3a) | Gantry + Racer (o3c, o3d) |
-|---|---|---|
-| Total bytes sent by all nodes | 13.9 TB (1.08x the bytes delivered) | 27.1 TB (2.1x) |
-| Bytes sent per node, p50 / p99 / max | 7.0 / 36.1 / 50.1 GB | 17.3 / 45.4 / 56.6 GB |
-| Peak fleet receive rate | 479 GB/s | 718-732 GB/s |
-| Time until network goes quiet | ~70s | ~50s |
+| | Gantry P2P 8 GiB | Racer 8 GiB | Gantry P2P 40 GB | Racer 40 GB |
+|---|---|---|---|---|
+| Total bytes sent by all nodes | 13.9 TB | 27.1 TB | 64.9 TB | 130.4 TB |
+| ... as a multiple of bytes delivered | 1.08x | 2.1x | 1.08x | 2.2x |
+| Bytes sent per node, p50 / p99 / max | 7.0 / 36.1 / 50.1 GB | 17.3 / 45.4 / 56.6 GB | 39-41 / 104-112 / 141-148 GB | 86 / 141-144 / 165 GB |
+| Peak fleet receive rate | 479 GB/s (10s) | 718-732 GB/s (10s) | 553-573 GB/s (30s) | 1109-1134 GB/s (30s) |
+| Time until network goes quiet | ~70s | ~50s | ~240s | ~150-180s |
+
+40 GB fleet receive rate per 30s window (GB/s):
+
+| Window end | 30s | 60s | 90s | 120s | 150s | 180s | 210s | 240s |
+|---|---|---|---|---|---|---|---|---|
+| Gantry 40g-c | 4.6 | 29 | 296 | 569 | 553 | 476 | 78 | 8 |
+| Gantry 40g-e | 4.7 | 25 | 282 | 552 | 573 | 481 | 82 | 14 |
+| Racer 40g-a | 670 | 1125 | 1109 | 831 | 280 | 19 | 3 | 0 |
+| Racer 40g-b | 631 | 1134 | 1114 | 964 | 171 | 16 | 0 | 0 |
+
+With 40 GB, data transfer ends well before the pulls do: about 180s (Racer) and 240s (gantry), against p50 of
+230s and 280s. The rest is containerd unpacking layers one at a time. Both modes pay that cost, so the gap
+between them is smaller than for 8 GiB in relative terms.
 
 ## What limits each mode
 
 **Gantry P2P**
-- Startup is slow. All nodes pull the layers in the same order, and the first copies come from the origin. With 3
-  origin replicas the fleet receive rate was 4 GB/s at 10s, 70 GB/s at 30s, and 480 GB/s at 50s.
-- Once peers have data, the fleet peaks at about 480 GB/s.
+- Startup is slow. All nodes pull the layers in the same order, and the first copies come from the origin. For
+  40 GB the fleet moved only 5-30 GB/s in the first 60s, while Racer moved 630-1130 GB/s.
+- Once peers have data, the fleet peaks at about 480-570 GB/s, and nodes send only what they receive (1.08x).
 
 **Gantry + Racer**
 - Each 16 MiB page has 3 owner nodes (`cmd/racer-dataplane/topology/src/placement.rs:10`, a protocol constant).
   Only owners serve a page to other nodes. A node that already holds the page does not serve it.
-- With 512 pages and 1497 nodes, some nodes own several pages. Each owned page means about 8 GB of upload, so
-  these nodes run their NIC at line rate (about 1.6 GB/s) for most of the run and set the tail.
+- With N pages and 1497 nodes, some nodes own several pages. Each owned page means about 8 GB of upload (40 GB:
+  up to 165 GB per node), so these nodes run their NIC at line rate for most of the run and set the tail.
 - Each node connects directly to at most 64 neighbors (`topology/src/lib.rs`, 32 rings). It reaches other owners
-  through a relay node, so every byte crosses the network about 2.1 times.
+  through a relay node, so every byte crosses the network about 2.1-2.2 times.
 - Fixing these needs design changes (non-owners serving pages, or fan-out through relays). No config knob changes
   them.
 
-## Changes made (draft PR, branch `bench/gantry-racer`)
+**Both:** containerd unpacks layers one at a time. With 40 GB images this is 50-100s of each pull.
+
+## Changes made (draft PR #1016, branch `bench/gantry-racer`)
 
 1. **Gantry: retry and resume Racer reads** (`internal/gantry/racer/racer.go`).
    - Before: when an owner is busy, Racer returns unavailable. Gantry turned that into a 503, or cut off a blob
@@ -91,19 +130,19 @@ Unpack adds about 6-8s. More parallel streams per node don't help.
    - Now gantry retries unavailable errors with backoff (150ms doubling to 2s, with jitter). It resumes a broken
      blob stream at the byte where it stopped, pinned to the same ETag.
    - New setting `racer_retry_budget` (`GANTRY_RACER_RETRY_BUDGET`, default 30s, 0 turns it off). The benchmark
-     used 120s.
+     used 120s. In the 40 GB runs, retries were rare (0-4 per run).
    - New metric `gantry_racer_mirror_retries_total{stage}`.
 2. **Racer: recover when the slab size changes** (`cmd/racer-dataplane/alloc/src/slab.rs`, `src/app.rs`).
    - Before: changing `RACER_SLAB_BYTES` made every slab-file node crash on start (`InvalidConfiguration`) until
      someone deleted the old files by hand.
    - Now slab files of the wrong size are recreated empty. Files of workers that no longer exist are deleted.
      Raw device mode is not changed.
-3. **Benchmark tools** in this directory.
+3. **Benchmark tools** in `hack/gantry-racer-bench/` (`bench.sh`, `genimage`, `nicstats.py`, `scrape.sh`,
+   `origin.yaml`, `diskprep.sh`). `bench.sh prune` was added for the 40 GB runs (see below).
 
 ## Racer config
 
-These values are set in ConfigMap `unbounded-system/racer-dataplane-config`. Byte and count values are
-node-wide.
+Set in ConfigMap `unbounded-system/racer-dataplane-config`. Byte and count values are node-wide.
 
 | Key | Value | Default | Why |
 |---|---|---|---|
@@ -128,6 +167,13 @@ D8ads_v5 and D8d_v5 nodes: `diskprep.sh` unmounts `/mnt` and wipes `/dev/sdb`. T
 
 ## Other findings
 
+- **Gantry P2P keeps pulled content on disk for an hour.** In `storage_mode: containerd`, gantry holds a
+  containerd lease on every blob it ingests for `containerd_lease_ttl` (default 60m,
+  `internal/gantry/containerdstore/lease.go:81`). Removing the image does not free the space, and kubelet image GC
+  can't reclaim it. Two 40 GB gantry pulls in a row filled the 123 GiB OS disk: 9 nodes hit DiskPressure and
+  kubelet evicted pods (including racer-dataplane). That run (40g-d) is not reported. `bench.sh prune` now deletes
+  gantry leases before each run. (Inference, not checked: Racer mode does not create these leases, since its
+  data lives in the Racer cache.)
 - Gantry P2P needs a direct path from containerd to the registry. Its mirror returns 503 on purpose when no peer
   has a layer yet (`internal/gantry/mirror/mirror.go`). Without the fallback, 18 of 1497 nodes failed.
 - Changing a Racer setting rolls the DaemonSet with `maxUnavailable: 1`, which takes more than 15 minutes on 1500
@@ -136,17 +182,10 @@ D8ads_v5 and D8d_v5 nodes: `diskprep.sh` unmounts `/mnt` and wipes `/dev/sdb`. T
   recovered.
 - Racer does not log request failures. Use `/debug/failures` on the metrics port (9090).
 
-## Running it
-
-```sh
-export BENCH_CONTEXT=$(kubectl config current-context)
-kubectl apply -f origin.yaml
-./bench.sh prep                      # hosts.toml on every worker
-./bench.sh diskprep                  # optional: free local disks for Racer
-ref=$(./bench.sh gen g8x1g-a 8 1073741824)
-NIC_SECS=100 ./bench.sh run gantry-a "$ref" crictl
-python3 nicstats.py ../../tmp/gantry-racer-bench/gantry-a.tsv 10
-```
+## Raw data
 
-Racer mode is on when a ClusterCache named `gantry` exists. Delete it to go back to gantry P2P. To keep the Racer
-DaemonSet while doing that, first create another ClusterCache, for example `bench-hold`.
+Per-run files (`<run>.tsv` per-node results and NIC samples, `<run>.log`, `<run>.{pre,post}.{gantry,racer}.prom`)
+are in `tmp/gantry-racer-bench/` (not committed). Run names there: `racer3-40g-a`, `racer3-40g-b`, `gantry3-40g-c`,
+`gantry3-40g-e`, `gantry3-8x1g-o3a`, `gantry3-8x1g-o3b`, `racer3-8x1g-o3c`, `racer3-8x1g-o3d`, `gantry3-1g-sa`,
+`gantry3-1g-sb`, `racer3-1g-sc`, `racer3-1g-sd`, `racer3-smoke`; single origin: `gantry-8x1g-k`, `gantry-8x1g-l`,
+`racer-8x1g-e`, `racer-8x1g-h`, `racer-8x1g-j`; fetch only: `racer-8x1g-f-curl`, `racer-8x1g-g-curl8`.

From 4b4a2a9351a7c9eac902e6d204684bcdefe91b96 Mon Sep 17 00:00:00 2001
From: Jordan Olshevski <jordan.olshevski@microsoft.com>
Date: Wed, 28 Oct 2026 23:26:38 +0000
Subject: [PATCH 16/28] hack/gantry-racer-bench: add license header and check
 errors in genimage

---
 hack/gantry-racer-bench/genimage/main.go | 21 ++++++++++++++++-----
 1 file changed, 16 insertions(+), 5 deletions(-)

diff --git a/hack/gantry-racer-bench/genimage/main.go b/hack/gantry-racer-bench/genimage/main.go
index cfd111d39..fcf85650c 100644
--- a/hack/gantry-racer-bench/genimage/main.go
+++ b/hack/gantry-racer-bench/genimage/main.go
@@ -1,3 +1,6 @@
+// Copyright (c) Microsoft Corporation.
+// SPDX-License-Identifier: Apache-2.0
+
 // Command genimage pushes an OCI image made of random, incompressible layers
 // to a plain HTTP registry. Every run uses a different seed so each benchmark
 // round starts from content that no cache has seen.
@@ -65,6 +68,7 @@ func main() {
 			defer wg.Done()
 
 			sem <- struct{}{}
+
 			defer func() { <-sem }()
 
 			results[i], errs[i] = pushLayer(*registry, *repo, i, *layerBytes, *seed)
@@ -210,7 +214,9 @@ func writeLayer(w io.Writer, diff hash.Hash, index int, size int64, seed uint64)
 			n = remaining
 		}
 
-		_, _ = rng.Read(buf[:n])
+		if _, err := rng.Read(buf[:n]); err != nil {
+			return err
+		}
 
 		if _, err := tw.Write(buf[:n]); err != nil {
 			return err
@@ -233,7 +239,7 @@ func pushBlob(registry, repo string, body io.Reader, _ http.Header) (string, err
 		return "", err
 	}
 
-	_ = resp.Body.Close()
+	_ = resp.Body.Close() //nolint:errcheck // best-effort close of an empty response body
 
 	if resp.StatusCode != http.StatusAccepted {
 		return "", fmt.Errorf("start upload: %s", resp.Status)
@@ -258,7 +264,7 @@ func pushBlob(registry, repo string, body io.Reader, _ http.Header) (string, err
 		return "", err
 	}
 
-	_ = resp.Body.Close()
+	_ = resp.Body.Close() //nolint:errcheck // best-effort close of an empty response body
 
 	if resp.StatusCode != http.StatusAccepted {
 		return "", fmt.Errorf("patch upload: %s", resp.Status)
@@ -306,10 +312,15 @@ func expect(resp *http.Response, err error) error {
 	if err != nil {
 		return err
 	}
-	defer resp.Body.Close()
+
+	defer func() { _ = resp.Body.Close() }() //nolint:errcheck // best-effort close
 
 	if resp.StatusCode/100 != 2 {
-		msg, _ := io.ReadAll(io.LimitReader(resp.Body, 4096))
+		msg, err := io.ReadAll(io.LimitReader(resp.Body, 4096))
+		if err != nil {
+			return fmt.Errorf("%s: read error body: %w", resp.Status, err)
+		}
+
 		return fmt.Errorf("%s: %s", resp.Status, msg)
 	}
 

From f0d59cebaa76ce5c290c006b7afd8c499e6e347e Mon Sep 17 00:00:00 2001
From: Jordan Olshevski <jordan.olshevski@microsoft.com>
Date: Wed, 28 Oct 2026 23:26:39 +0000
Subject: [PATCH 17/28] racer-loadgen: disable mirror retries in the
 single-attempt gantry test

---
 cmd/racer-loadgen/gantry_test.go | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/cmd/racer-loadgen/gantry_test.go b/cmd/racer-loadgen/gantry_test.go
index 9832f1d5c..7641396d5 100644
--- a/cmd/racer-loadgen/gantry_test.go
+++ b/cmd/racer-loadgen/gantry_test.go
@@ -103,7 +103,9 @@ func TestGantryIntegration(t *testing.T) {
 		return metadata, body, err
 	})
 
-	server := httptest.NewServer(mirror.New(cfg, nil, upstream, mirror.WithContentBackend(gantryracer.NewHandler(client, upstream, nil))).Handler())
+	// Retries are disabled so callback counts reflect exactly one attempt per page.
+	handler := gantryracer.NewHandler(client, upstream, nil, gantryracer.WithRetryBudget(0))
+	server := httptest.NewServer(mirror.New(cfg, nil, upstream, mirror.WithContentBackend(handler)).Handler())
 	t.Cleanup(server.Close)
 	opts := pullTestOptions(server.URL)
 	opts.Namespace = "loadgen.invalid"

From 172e99b80b2a0f60d4388ced90d58917ad26bd53 Mon Sep 17 00:00:00 2001
From: Jordan Olshevski <jordan.olshevski@microsoft.com>
Date: Wed, 28 Oct 2026 23:26:59 +0000
Subject: [PATCH 18/28] hack/gantry-racer-bench: close the NIC samples file in
 nicstats

---
 hack/gantry-racer-bench/nicstats.py | 27 ++++++++++++++-------------
 1 file changed, 14 insertions(+), 13 deletions(-)

diff --git a/hack/gantry-racer-bench/nicstats.py b/hack/gantry-racer-bench/nicstats.py
index 01349f9b4..66175d157 100644
--- a/hack/gantry-racer-bench/nicstats.py
+++ b/hack/gantry-racer-bench/nicstats.py
@@ -25,19 +25,20 @@ def main():
     step = int(sys.argv[2]) if len(sys.argv) > 2 else 5
     nodes = {}
     skipped = 0
-    for line in open(path):
-        f = line.split()
-        if not f or f[0] != "NIC":
-            continue
-        samples = []
-        try:
-            for s in f[2:]:
-                i, tx, rx = s.split(":")
-                samples.append((int(i), int(tx), int(rx)))
-        except ValueError:
-            skipped += 1
-            continue
-        nodes[f[1]] = samples
+    with open(path) as fh:
+        for line in fh:
+            f = line.split()
+            if not f or f[0] != "NIC":
+                continue
+            samples = []
+            try:
+                for s in f[2:]:
+                    i, tx, rx = s.split(":")
+                    samples.append((int(i), int(tx), int(rx)))
+            except ValueError:
+                skipped += 1
+                continue
+            nodes[f[1]] = samples
     if not nodes:
         sys.exit("no NIC lines")
     if skipped:

From b65aadfcb7a754777c1a16c2b70057d09bc8c432 Mon Sep 17 00:00:00 2001
From: Jordan Olshevski <jordan.olshevski@microsoft.com>
Date: Wed, 28 Oct 2026 23:34:50 +0000
Subject: [PATCH 19/28] gantry/racer: count only backoff against the retry
 budget

---
 internal/gantry/racer/racer.go      | 20 ++++++++++++--------
 internal/gantry/racer/retry_test.go | 20 ++++++++++++++++++--
 2 files changed, 30 insertions(+), 10 deletions(-)

diff --git a/internal/gantry/racer/racer.go b/internal/gantry/racer/racer.go
index a263ce43e..f1442a152 100644
--- a/internal/gantry/racer/racer.go
+++ b/internal/gantry/racer/racer.go
@@ -682,20 +682,22 @@ type retryPolicy struct {
 	observe           func(stage string)
 }
 
-// retryState tracks one request's retry budget across all stages.
+// retryState tracks one request's retry budget across all stages. Only
+// backoff counts against the budget, so time spent successfully streaming a
+// long body does not prevent resuming it after a later transient failure.
 type retryState struct {
-	policy   retryPolicy
-	deadline time.Time
-	attempt  int
+	policy  retryPolicy
+	spent   time.Duration
+	attempt int
 }
 
 func (p retryPolicy) start() *retryState {
-	return &retryState{policy: p, deadline: time.Now().Add(p.budget)}
+	return &retryState{policy: p}
 }
 
 // wait sleeps for a jittered exponential backoff and reports whether the
-// caller should retry. It refuses when the sleep would exceed the budget or
-// the request context ends.
+// caller should retry. It refuses when the sleep would push the total backoff
+// past the budget or the request context ends.
 func (r *retryState) wait(ctx context.Context, stage string) bool {
 	if r.policy.budget <= 0 || ctx.Err() != nil {
 		return false
@@ -710,10 +712,12 @@ func (r *retryState) wait(ctx context.Context, stage string) bool {
 	// Equal jitter keeps a floor while spreading synchronized retries.
 	delay := backoff/2 + rand.N(backoff/2+1) //nolint:gosec // jitter, not security
 
-	if time.Now().Add(delay).After(r.deadline) {
+	if r.spent+delay > r.policy.budget {
 		return false
 	}
 
+	r.spent += delay
+
 	if r.policy.observe != nil {
 		r.policy.observe(stage)
 	}
diff --git a/internal/gantry/racer/retry_test.go b/internal/gantry/racer/retry_test.go
index 19dbf7442..90c628109 100644
--- a/internal/gantry/racer/retry_test.go
+++ b/internal/gantry/racer/retry_test.go
@@ -431,17 +431,33 @@ func TestRacerRetryMetrics(t *testing.T) {
 func TestRacerRetryWaitBudget(t *testing.T) {
 	var stages retryStages
 
-	r := &retryState{policy: retryPolicy{budget: time.Second, base: time.Millisecond, cap: time.Millisecond, observe: stages.observe}, deadline: time.Now().Add(time.Second)}
+	r := retryPolicy{budget: time.Second, base: time.Millisecond, cap: time.Millisecond, observe: stages.observe}.start()
 	if !r.wait(context.Background(), "get") || stages.count("get") != 1 {
 		t.Fatal("expected retry")
 	}
 
-	expired := &retryState{policy: r.policy, deadline: time.Now()}
+	expired := &retryState{policy: r.policy, spent: time.Second}
 	if expired.wait(context.Background(), "get") {
 		t.Fatal("retried past budget")
 	}
 }
 
+// The budget bounds backoff, not elapsed time: a long successful transfer
+// must still be able to retry its first transient failure.
+func TestRacerRetryBudgetExcludesElapsedTime(t *testing.T) {
+	r := retryPolicy{budget: 5 * time.Millisecond, base: time.Millisecond, cap: time.Millisecond}.start()
+
+	time.Sleep(20 * time.Millisecond)
+
+	if !r.wait(context.Background(), "write_body") {
+		t.Fatal("elapsed time without backoff consumed the retry budget")
+	}
+
+	if r.spent <= 0 || r.spent > time.Millisecond {
+		t.Fatalf("spent = %v, want (0, 1ms]", r.spent)
+	}
+}
+
 // Resume through WrapHTTP over real connections, including the splice and
 // userspace fallback paths, so leftover SDK write deadlines cannot break it.
 func TestRacerRetryResumesOverHTTP(t *testing.T) {

From 6f822fa3cb347ee55e9879df009bf62f18fea474 Mon Sep 17 00:00:00 2001
From: Jordan Olshevski <jordan.olshevski@microsoft.com>
Date: Wed, 28 Oct 2026 23:35:45 +0000
Subject: [PATCH 20/28] gantry/racer: share one retry budget across Stat, Get,
 and resume

---
 internal/gantry/racer/racer.go      | 11 +++++----
 internal/gantry/racer/retry_test.go | 36 +++++++++++++++++++++++++++++
 2 files changed, 42 insertions(+), 5 deletions(-)

diff --git a/internal/gantry/racer/racer.go b/internal/gantry/racer/racer.go
index f1442a152..29836f4b6 100644
--- a/internal/gantry/racer/racer.go
+++ b/internal/gantry/racer/racer.go
@@ -487,13 +487,14 @@ func (s *Handler) ServeContent(w http.ResponseWriter, r *http.Request, ref iface
 		return
 	}
 
-	metadata, options, done := s.prepareResponse(w, r, id, ref, request, offset)
+	// One retry budget covers Stat, Get, and body resumption for this request.
+	retry := s.retry.start()
+
+	metadata, options, done := s.prepareResponse(w, r, id, ref, request, offset, retry)
 	if done {
 		return
 	}
 
-	retry := s.retry.start()
-
 	value, err := s.getWithRetry(r.Context(), retry, request, options...)
 	if err != nil {
 		expected := int64(-1)
@@ -538,7 +539,7 @@ func (s *Handler) ServeContent(w http.ResponseWriter, r *http.Request, ref iface
 // prepareResponse handles HEAD and validates the snapshot needed for blob resume.
 // A true done result means it already wrote a final response; no Object is open.
 // Full GETs get metadata from Get instead of issuing a redundant Stat.
-func (s *Handler) prepareResponse(w http.ResponseWriter, r *http.Request, id string, ref ifaces.OriginRef, request racersdk.Request, offset int64) (racersdk.Metadata, []racersdk.ReadOptions, bool) {
+func (s *Handler) prepareResponse(w http.ResponseWriter, r *http.Request, id string, ref ifaces.OriginRef, request racersdk.Request, offset int64, retry *retryState) (racersdk.Metadata, []racersdk.ReadOptions, bool) {
 	if r.Method != http.MethodHead && offset == 0 {
 		if ref.Kind == ifaces.KindManifest {
 			return racersdk.Metadata{}, []racersdk.ReadOptions{{SmallObject: true}}, false
@@ -547,7 +548,7 @@ func (s *Handler) prepareResponse(w http.ResponseWriter, r *http.Request, id str
 		return racersdk.Metadata{}, nil, false
 	}
 
-	metadata, err := s.statWithRetry(r.Context(), s.retry.start(), request)
+	metadata, err := s.statWithRetry(r.Context(), retry, request)
 	if err != nil {
 		s.logRacerFailure(id, "stat", err, -1, 0)
 		s.racerError(w, r, ref, err)
diff --git a/internal/gantry/racer/retry_test.go b/internal/gantry/racer/retry_test.go
index 90c628109..2007f4d0d 100644
--- a/internal/gantry/racer/retry_test.go
+++ b/internal/gantry/racer/retry_test.go
@@ -205,6 +205,42 @@ func TestRacerRetryStatTransientThenSuccess(t *testing.T) {
 	}
 }
 
+// A ranged GET retries Stat and Get against one shared budget, so the Stat
+// retry leaves no room for a Get retry.
+func TestRacerRetryRangedGetSharesBudget(t *testing.T) {
+	data := []byte("abc")
+	client := &retryClient{
+		Client: racerFakeClient(t, staticOrigin(t, data)),
+		statErr: func(call int) error {
+			if call == 1 {
+				return unavailable()
+			}
+
+			return nil
+		},
+		getErr: func(int, []racersdk.ReadOptions) error { return unavailable() },
+	}
+
+	var stages retryStages
+
+	// Each delay is in [1ms, 2ms], so one retry fits the budget and two do not.
+	h := retryHandler(client, 2*time.Millisecond, &stages)
+	h.retry.base = 2 * time.Millisecond
+	h.retry.cap = 2 * time.Millisecond
+
+	ref := retryRef(racerDigest(data))
+	ref.Offset = 1
+
+	rec, aborted := serveRetry(h, httptest.NewRequest(http.MethodGet, "/", nil), ref)
+	if aborted || rec.Code == http.StatusOK || rec.Code == http.StatusPartialContent {
+		t.Fatalf("aborted=%v status=%d", aborted, rec.Code)
+	}
+
+	if client.stats != 2 || stages.count("stat") != 1 || client.gets != 1 || stages.count("get") != 0 {
+		t.Fatalf("stats=%d stat retries=%d gets=%d get retries=%d", client.stats, stages.count("stat"), client.gets, stages.count("get"))
+	}
+}
+
 func TestRacerRetryGivesUpAfterBudget(t *testing.T) {
 	data := []byte("abc")
 	client := &retryClient{

From cdead54ad00b02609c722d0eaec14a7b7329376e Mon Sep 17 00:00:00 2001
From: Jordan Olshevski <jordan.olshevski@microsoft.com>
Date: Wed, 28 Oct 2026 23:36:39 +0000
Subject: [PATCH 21/28] hack/gantry-racer-bench: fail gen when any origin push
 fails

---
 hack/gantry-racer-bench/bench.sh | 26 ++++++++++++++++++++------
 1 file changed, 20 insertions(+), 6 deletions(-)

diff --git a/hack/gantry-racer-bench/bench.sh b/hack/gantry-racer-bench/bench.sh
index 8768f2d63..1d0ed0654 100755
--- a/hack/gantry-racer-bench/bench.sh
+++ b/hack/gantry-racer-bench/bench.sh
@@ -252,12 +252,26 @@ cmd_gen() {
 	tools=$(tools_pod)
 	seed=$(od -An -N8 -tu8 /dev/urandom | tr -d ' ')
 	created=$(date -u +%Y-%m-%dT%H:%M:%SZ)
-	refs=$(for ip in $(origin_ips); do
-		kubectl -n "$NS" exec "$tools" -- /tmp/genimage -registry "http://$ip:5000" \
-			-repo "bench/$name" -layers "$layers" -layer-bytes "$bytes" -parallel 8 \
-			-seed "$seed" -created "$created" &
-	done | sort -u)
-	wait
+	# Wait for every push inside the substitution: the parent shell cannot
+	# observe jobs started there, and a failed push prints no reference.
+	refs=$(
+		pids=()
+		for ip in $(origin_ips); do
+			kubectl -n "$NS" exec "$tools" -- /tmp/genimage -registry "http://$ip:5000" \
+				-repo "bench/$name" -layers "$layers" -layer-bytes "$bytes" -parallel 8 \
+				-seed "$seed" -created "$created" &
+			pids+=($!)
+		done
+		rc=0
+		for pid in "${pids[@]}"; do
+			wait "$pid" || rc=1
+		done
+		exit "$rc"
+	) || {
+		echo "image push to an origin replica failed" >&2
+		exit 1
+	}
+	refs=$(printf '%s\n' "$refs" | sort -u)
 	if [ "$(printf '%s\n' "$refs" | grep -c .)" -ne 1 ]; then
 		echo "origin replicas disagree: $refs" >&2
 		exit 1

From 8e7b3f05380ffa96f191e4e75a8aa9d4ec1cad2e Mon Sep 17 00:00:00 2001
From: Jordan Olshevski <jordan.olshevski@microsoft.com>
Date: Wed, 28 Oct 2026 23:37:19 +0000
Subject: [PATCH 22/28] hack/gantry-racer-bench: fail scrape when any pod is
 not scraped

---
 hack/gantry-racer-bench/scrape.sh | 32 +++++++++++++++++++++++++++----
 1 file changed, 28 insertions(+), 4 deletions(-)

diff --git a/hack/gantry-racer-bench/scrape.sh b/hack/gantry-racer-bench/scrape.sh
index 9c9c2d977..e9460c2d3 100755
--- a/hack/gantry-racer-bench/scrape.sh
+++ b/hack/gantry-racer-bench/scrape.sh
@@ -1,6 +1,7 @@
 #!/usr/bin/env bash
 # Sum Prometheus counters across all gantry (:9095) and racer-dataplane (:9090)
-# pods. Runs the scrape from the tools pod in gantry-bench.
+# pods. Runs the scrape from the tools pod in gantry-bench. Fails rather than
+# writing a partial fleet total when any pod cannot be scraped.
 # Usage: scrape.sh <out-prefix>   -> <out-prefix>.gantry.prom, <out-prefix>.racer.prom
 set -euo pipefail
 out=$1
@@ -10,10 +11,33 @@ scrape() {
   local label=$1 port=$2 file=$3
   kubectl -n $ns get pods -l app.kubernetes.io/name=$label -o jsonpath='{range .items[*]}{.status.podIP}{"\n"}{end}' >"$file.ips"
   kubectl -n gantry-bench cp "$file.ips" "$tools:/tmp/$label.ips" >/dev/null
-  kubectl -n gantry-bench exec "$tools" -- sh -c "rm -rf /tmp/m-$label; mkdir -p /tmp/m-$label; for ip in \$(cat /tmp/$label.ips); do wget -qO /tmp/m-$label/\$ip -T 5 http://\$ip:$port/metrics & if [ \$(jobs | wc -l) -ge 64 ]; then wait; fi; done; wait; cat /tmp/m-$label/*" 2>/dev/null |
-    grep -v '^#' | sed -E 's/(disk|worker|device)="[^"]*",?//g; s/\{\}//' | awk 'NF==2 {s[$1]+=$2} END {for (k in s) printf "%s %.0f\n", k, s[k]}' | sort >"$file"
   rm -f "$file.ips"
+  # Each download gets its final name only on success, so the file count
+  # shows whether every pod was scraped.
+  if ! kubectl -n gantry-bench exec "$tools" -- sh -c "rm -rf /tmp/m-$label; mkdir -p /tmp/m-$label
+for ip in \$(cat /tmp/$label.ips); do
+  (wget -qO /tmp/m-$label/\$ip.part -T 5 http://\$ip:$port/metrics && mv /tmp/m-$label/\$ip.part /tmp/m-$label/\$ip.prom) &
+  if [ \$(jobs | wc -l) -ge 64 ]; then wait; fi
+done
+wait
+want=\$(grep -c . /tmp/$label.ips)
+got=\$(ls /tmp/m-$label | grep -c '\.prom\$')
+if [ \"\$got\" -ne \"\$want\" ]; then
+  echo \"$label: scraped \$got of \$want pods\" >&2
+  exit 1
+fi
+if [ \"\$want\" -gt 0 ]; then cat /tmp/m-$label/*.prom; fi" >"$file.raw"; then
+    rm -f "$file.raw"
+    return 1
+  fi
+  { grep -v '^#' "$file.raw" || true; } | sed -E 's/(disk|worker|device)="[^"]*",?//g; s/\{\}//' | awk 'NF==2 {s[$1]+=$2} END {for (k in s) printf "%s %.0f\n", k, s[k]}' | sort >"$file"
+  rm -f "$file.raw"
 }
 scrape gantry 9095 "$out.gantry.prom" &
+gantry=$!
 scrape racer-dataplane 9090 "$out.racer.prom" &
-wait
+racer=$!
+rc=0
+wait "$gantry" || rc=1
+wait "$racer" || rc=1
+exit "$rc"

From f1706d81ddafb79aecc294df5421743d9ac12a0a Mon Sep 17 00:00:00 2001
From: Jordan Olshevski <jordan.olshevski@microsoft.com>
Date: Wed, 28 Oct 2026 23:37:53 +0000
Subject: [PATCH 23/28] hack/gantry-racer-bench: wait for prune and diskprep
 pods to finish

---
 hack/gantry-racer-bench/bench.sh | 29 +++++++++++++++++++++++++----
 1 file changed, 25 insertions(+), 4 deletions(-)

diff --git a/hack/gantry-racer-bench/bench.sh b/hack/gantry-racer-bench/bench.sh
index 1d0ed0654..6accd1a0e 100755
--- a/hack/gantry-racer-bench/bench.sh
+++ b/hack/gantry-racer-bench/bench.sh
@@ -62,6 +62,29 @@ worker_affinity() {
 EOF
 }
 
+# ds_records waits until every pod of DaemonSet <app> has logged a line that
+# starts with <prefix>, or <timeout> seconds pass, and prints the first such
+# line from each pod. The pods have no readiness probe, so rollout status alone
+# returns before their work is done.
+ds_records() {
+	local app=$1 prefix=$2 timeout=$3 deadline pods records
+	deadline=$(($(date +%s) + timeout))
+	while :; do
+		pods=$(kubectl -n "$NS" get pods -l "app=$app" -o name)
+		records=$(printf '%s\n' "$pods" | grep . |
+			xargs -r -P 64 -I{} sh -c "kubectl -n $NS logs {} 2>/dev/null | grep -m 1 '^$prefix' || true") || true
+		if [ "$(printf '%s\n' "$records" | grep -c .)" -ge "$(printf '%s\n' "$pods" | grep -c .)" ]; then
+			break
+		fi
+		if [ "$(date +%s)" -ge "$deadline" ]; then
+			echo "warning: $(printf '%s\n' "$records" | grep -c .) of $(printf '%s\n' "$pods" | grep -c .) $app pods reported $prefix" >&2
+			break
+		fi
+		sleep 10
+	done
+	printf '%s\n' "$records" | grep . || true
+}
+
 cmd_prep() {
 	kubectl apply -f - <<EOF
 apiVersion: apps/v1
@@ -184,8 +207,7 @@ $(worker_affinity)
             path: /
 EOF
 	kubectl -n "$NS" rollout status ds/prune --timeout=15m
-	kubectl -n "$NS" get pods -l app=prune -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}' |
-		xargs -P 64 -I{} kubectl -n "$NS" logs {} 2>/dev/null | grep '^PRUNED' | sort -k3 -n | tail -n 5
+	ds_records prune PRUNED 900 | sort -k3 -n | tail -n 5
 	kubectl -n "$NS" delete ds prune --wait=false >/dev/null
 }
 
@@ -236,8 +258,7 @@ $(sed 's/^/              /' "$DIR/diskprep.sh")
             path: /
 EOF
 	kubectl -n "$NS" rollout status ds/diskprep --timeout=15m
-	kubectl -n "$NS" get pods -l app=diskprep -o name |
-		xargs -P 64 -I{} sh -c "kubectl -n $NS logs {} | grep '^DISK' || true" >"$OUT/disks.txt"
+	ds_records diskprep DISK 900 >"$OUT/disks.txt"
 	awk '$3 != "none" {print $2, $3}' "$OUT/disks.txt" |
 		xargs -P 64 -n 2 sh -c 'kubectl annotate node "$0" --overwrite "racer.unbounded-cloud.io/block-devices=^$1\$" >/dev/null'
 	awk '{print $3 == "none" ? $4 : "device"}' "$OUT/disks.txt" | sort | uniq -c

From 7f629e3610288ae4d84dfa243c134eeb7924bf5a Mon Sep 17 00:00:00 2001
From: Jordan Olshevski <jordan.olshevski@microsoft.com>
Date: Wed, 28 Oct 2026 23:52:06 +0000
Subject: [PATCH 24/28] racer: lock stale slab files before pruning them

---
 cmd/racer-dataplane/src/app.rs       | 42 +++++++++++++++++++++++++++-
 cmd/racer-dataplane/src/app/tests.rs | 28 +++++++++++++++++++
 2 files changed, 69 insertions(+), 1 deletion(-)

diff --git a/cmd/racer-dataplane/src/app.rs b/cmd/racer-dataplane/src/app.rs
index 3efb034c8..5e77e2bb0 100644
--- a/cmd/racer-dataplane/src/app.rs
+++ b/cmd/racer-dataplane/src/app.rs
@@ -246,8 +246,15 @@ impl NodeState {
 /// Remove slab files left by workers that no longer exist after a worker-count change.
 ///
 /// Only regular files named `worker-<id>-slab-0.dat` with an id outside `workers` are
-/// removed. Failures are logged and ignored because stale files never affect correctness.
+/// removed. Each file is removed only while holding the same exclusive lock that slab
+/// startup takes, and only if the path still names the locked inode, so a concurrent
+/// instance's live slabs are skipped. Failures are logged and ignored because stale
+/// files never affect correctness.
 fn prune_stale_slab_files(directory: &std::path::Path, workers: &[WorkerId]) {
+    use std::os::fd::AsRawFd;
+    use std::os::unix::fs::MetadataExt;
+    use std::os::unix::fs::OpenOptionsExt;
+
     let Ok(entries) = std::fs::read_dir(directory) else {
         return;
     };
@@ -265,6 +272,39 @@ fn prune_stale_slab_files(directory: &std::path::Path, workers: &[WorkerId]) {
             continue;
         }
         let path = entry.path();
+        let file = match std::fs::OpenOptions::new()
+            .read(true)
+            .custom_flags(libc::O_NOFOLLOW)
+            .open(&path)
+        {
+            Ok(file) => file,
+            Err(error) => {
+                eprintln!(
+                    "racer-dataplane: stage=slabs failed to open stale slab file {}: {error}",
+                    path.display()
+                );
+                continue;
+            }
+        };
+        // SAFETY: flock synchronously borrows this live descriptor.
+        if unsafe { libc::flock(file.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) } != 0 {
+            eprintln!(
+                "racer-dataplane: stage=slabs kept locked stale slab file {}: {}",
+                path.display(),
+                std::io::Error::last_os_error()
+            );
+            continue;
+        }
+        let same = match (file.metadata(), std::fs::symlink_metadata(&path)) {
+            (Ok(locked), Ok(current)) => {
+                current.is_file() && locked.dev() == current.dev() && locked.ino() == current.ino()
+            }
+            _ => false,
+        };
+        if !same {
+            continue;
+        }
+        // The lock is held until `file` drops, after the unlink.
         match std::fs::remove_file(&path) {
             Ok(()) => eprintln!(
                 "racer-dataplane: stage=slabs removed stale slab file {}",
diff --git a/cmd/racer-dataplane/src/app/tests.rs b/cmd/racer-dataplane/src/app/tests.rs
index bb656fa81..62d9ae532 100644
--- a/cmd/racer-dataplane/src/app/tests.rs
+++ b/cmd/racer-dataplane/src/app/tests.rs
@@ -4922,3 +4922,31 @@ fn prune_stale_slab_files_removes_only_extra_worker_files() {
     super::prune_stale_slab_files(&directory.join("missing"), &[WorkerId(0)]);
     std::fs::remove_dir_all(&directory).unwrap();
 }
+
+/// A stale slab file still locked by another instance is kept.
+#[test]
+fn prune_stale_slab_files_skips_locked_files() {
+    use std::os::fd::AsRawFd;
+    let directory = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR"))
+        .join("target")
+        .join(format!("prune-locked-slabs-test-{}", std::process::id()));
+    let _ = std::fs::remove_dir_all(&directory);
+    std::fs::create_dir_all(&directory).unwrap();
+    let locked = directory.join("worker-3-slab-0.dat");
+    let unlocked = directory.join("worker-4-slab-0.dat");
+    std::fs::write(&locked, b"x").unwrap();
+    std::fs::write(&unlocked, b"x").unwrap();
+    let holder = std::fs::File::open(&locked).unwrap();
+    // SAFETY: flock borrows a live descriptor owned by this test.
+    assert_eq!(
+        unsafe { libc::flock(holder.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
+        0
+    );
+    super::prune_stale_slab_files(&directory, &[WorkerId(0)]);
+    assert!(locked.exists());
+    assert!(!unlocked.exists());
+    drop(holder);
+    super::prune_stale_slab_files(&directory, &[WorkerId(0)]);
+    assert!(!locked.exists());
+    std::fs::remove_dir_all(&directory).unwrap();
+}

From 9ff5accbf6d4abf15fe9fea3e880b7f63a571a68 Mon Sep 17 00:00:00 2001
From: Jordan Olshevski <jordan.olshevski@microsoft.com>
Date: Wed, 28 Oct 2026 23:59:51 +0000
Subject: [PATCH 25/28] hack/gantry-racer-bench: fail origin_tx when any
 replica scrape fails

---
 hack/gantry-racer-bench/bench.sh | 24 +++++++++++++++++-------
 1 file changed, 17 insertions(+), 7 deletions(-)

diff --git a/hack/gantry-racer-bench/bench.sh b/hack/gantry-racer-bench/bench.sh
index 6accd1a0e..12e068115 100755
--- a/hack/gantry-racer-bench/bench.sh
+++ b/hack/gantry-racer-bench/bench.sh
@@ -36,13 +36,20 @@ origin_ips() {
 	kubectl -n "$NS" get pod -l app=origin -o jsonpath='{range .items[*]}{.status.podIP}{"\n"}{end}'
 }
 
-# origin_tx sums network egress over all origin replicas.
+# origin_tx sums network egress over all origin replicas. It fails if any
+# replica cannot be scraped, rather than printing a partial total. Callers use
+# command substitution, where errexit does not apply, so failures are explicit.
 origin_tx() {
-	local tools ip
-	tools=$(tools_pod)
+	local tools ip metrics=""
+	tools=$(tools_pod) || return 1
 	for ip in $(origin_ips); do
-		kubectl -n "$NS" exec "$tools" -- wget -qO- "http://$ip:5001/metrics"
-	done | awk '$1 == "process_network_transmit_bytes_total" {s += $2} END {printf "%.0f\n", s}'
+		if ! metrics+=$(kubectl -n "$NS" exec "$tools" -- wget -qO- "http://$ip:5001/metrics"); then
+			echo "origin metrics scrape failed for $ip" >&2
+			return 1
+		fi
+		metrics+=$'\n'
+	done
+	printf '%s' "$metrics" | awk '$1 == "process_network_transmit_bytes_total" {s += $2} END {printf "%.0f\n", s}'
 }
 
 tools_pod() {
@@ -458,8 +465,11 @@ EOF
 
 	mv "$results.tmp" "$results"
 	local tx1
-	tx1=$(origin_tx)
-	echo "ORIGIN_TX $((tx1 - tx0))" >>"$results"
+	if tx1=$(origin_tx); then
+		echo "ORIGIN_TX $((tx1 - tx0))" >>"$results"
+	else
+		echo "origin egress unavailable; omitting ORIGIN_TX" >&2
+	fi
 	late=$(awk -v s="$start_at" '$1 == "READY" && $3 > s' "$results" | wc -l)
 	echo "pods ready after start: $late" >&2
 	kubectl -n "$NS" delete ds "$name" --wait=false >/dev/null

From 8cf702abf78c10567bfa689560bcd441426add82 Mon Sep 17 00:00:00 2001
From: Jordan Olshevski <jordan.olshevski@microsoft.com>
Date: Wed, 28 Oct 2026 23:59:55 +0000
Subject: [PATCH 26/28] hack/gantry-racer-bench: skip node annotation when no
 raw disk is found

---
 hack/gantry-racer-bench/bench.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/hack/gantry-racer-bench/bench.sh b/hack/gantry-racer-bench/bench.sh
index 12e068115..aced4b9f9 100755
--- a/hack/gantry-racer-bench/bench.sh
+++ b/hack/gantry-racer-bench/bench.sh
@@ -267,7 +267,7 @@ EOF
 	kubectl -n "$NS" rollout status ds/diskprep --timeout=15m
 	ds_records diskprep DISK 900 >"$OUT/disks.txt"
 	awk '$3 != "none" {print $2, $3}' "$OUT/disks.txt" |
-		xargs -P 64 -n 2 sh -c 'kubectl annotate node "$0" --overwrite "racer.unbounded-cloud.io/block-devices=^$1\$" >/dev/null'
+		xargs -r -P 64 -n 2 sh -c 'kubectl annotate node "$0" --overwrite "racer.unbounded-cloud.io/block-devices=^$1\$" >/dev/null'
 	awk '{print $3 == "none" ? $4 : "device"}' "$OUT/disks.txt" | sort | uniq -c
 	kubectl -n "$NS" delete ds diskprep --wait=false >/dev/null
 }

From 3d3792b777a4d58c093f42d36265d7dfef897c99 Mon Sep 17 00:00:00 2001
From: Jordan Olshevski <jordan.olshevski@microsoft.com>
Date: Thu, 29 Oct 2026 00:00:14 +0000
Subject: [PATCH 27/28] hack/gantry-racer-bench: count pods without an IP in
 scrape

---
 hack/gantry-racer-bench/scrape.sh | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/hack/gantry-racer-bench/scrape.sh b/hack/gantry-racer-bench/scrape.sh
index e9460c2d3..30892043a 100755
--- a/hack/gantry-racer-bench/scrape.sh
+++ b/hack/gantry-racer-bench/scrape.sh
@@ -13,14 +13,15 @@ scrape() {
   kubectl -n gantry-bench cp "$file.ips" "$tools:/tmp/$label.ips" >/dev/null
   rm -f "$file.ips"
   # Each download gets its final name only on success, so the file count
-  # shows whether every pod was scraped.
+  # shows whether every pod was scraped. The IP file has one line per pod,
+  # blank for a pod without an IP yet, so count every line.
   if ! kubectl -n gantry-bench exec "$tools" -- sh -c "rm -rf /tmp/m-$label; mkdir -p /tmp/m-$label
 for ip in \$(cat /tmp/$label.ips); do
   (wget -qO /tmp/m-$label/\$ip.part -T 5 http://\$ip:$port/metrics && mv /tmp/m-$label/\$ip.part /tmp/m-$label/\$ip.prom) &
   if [ \$(jobs | wc -l) -ge 64 ]; then wait; fi
 done
 wait
-want=\$(grep -c . /tmp/$label.ips)
+want=\$(grep -c '' /tmp/$label.ips)
 got=\$(ls /tmp/m-$label | grep -c '\.prom\$')
 if [ \"\$got\" -ne \"\$want\" ]; then
   echo \"$label: scraped \$got of \$want pods\" >&2

From fbd2a7b9b05832aa2f7c1f6d3e1bc0faa614b2d0 Mon Sep 17 00:00:00 2001
From: Jordan Olshevski <jordan.olshevski@microsoft.com>
Date: Thu, 29 Oct 2026 00:00:36 +0000
Subject: [PATCH 28/28] racer: describe checkpoint handling after a slab resize
 rollback

---
 cmd/racer-dataplane/src/app.rs | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/cmd/racer-dataplane/src/app.rs b/cmd/racer-dataplane/src/app.rs
index 5e77e2bb0..b6a23d965 100644
--- a/cmd/racer-dataplane/src/app.rs
+++ b/cmd/racer-dataplane/src/app.rs
@@ -1570,9 +1570,12 @@ impl WorkerApplication {
                 config.segment_bytes,
                 crate::model::PAGE_BYTES as usize + crate::store::MAX_HEADER_BYTES + 16,
             )
-            // Slab files are a re-fetchable cache. A changed slab size makes any
-            // checkpoint for this file incompatible (its recorded slab_bytes no
-            // longer match), so recovery starts this worker cold.
+            // Slab files are a re-fetchable cache. A checkpoint taken at the old
+            // size no longer matches slab_bytes, so recovery starts this worker
+            // cold. An older checkpoint taken at the new size, as after a resize
+            // rollback, may still be accepted; record validation and AEAD then
+            // turn its references to discarded pages into misses, as for any
+            // checkpoint that outlives its payload.
             .reset_mismatched_size()
         });
         let reader = Rc::new(