From ce6233adc3dbaba138ca74fed691af9d2bf0d9fc Mon Sep 17 00:00:00 2001 From: Daniel Frenkel Date: Thu, 12 Nov 2026 20:12:58 -0800 Subject: [PATCH 1/6] predictions: bench heartbeat file retry on Windows (registered before the change) Co-Authored-By: Claude Opus 5.5 --- .../2026-11-bench-heartbeat-retry.md | 47 +++++++++++++++++++ 1 file changed, 47 insertions(+) create mode 100644 tools/regression-gate/predictions/2026-11-bench-heartbeat-retry.md diff --git a/tools/regression-gate/predictions/2026-11-bench-heartbeat-retry.md b/tools/regression-gate/predictions/2026-11-bench-heartbeat-retry.md new file mode 100644 index 0000000..3852fd2 --- /dev/null +++ b/tools/regression-gate/predictions/2026-11-bench-heartbeat-retry.md @@ -0,0 +1,47 @@ +# Bench tools: open guard heartbeat file on Windows — predictions + +Written and committed before the code change, on branch `bench/heartbeat-retry` (base: origin/main +`96674ca`). No check below had been run when this file was committed. + +## What happened (TP-1 sitting, 2026-11-12, 20:02 PT, sealed unit, old carriage) + +About 70 s after open_guard.py started, and 10 s into tp1_run.py, open_guard.py exited with +`PermissionError: [WinError 5] Access is denied` from `os.replace(guard-heartbeat.tmp, +guard-heartbeat.json)` in `bench_common.write_heartbeat`. On Windows, replacing a file fails while +another process has it open, and tp1_run.py reads that file before and during every move. With the +guard gone, its heartbeat went stale; tp1_run.py sent STOP mid-move (the 3.6 s stale reading) and +ended INCOMPLETE. The interlock worked; the run gave no TP-1 reading (instrument failure, disclosed). +The reader has the same exposure: `guard_ok` turns a read that collides with a replace into "no +heartbeat", which would also stop a move. +Daniel (2026-11-12, question tool): fix now, register it, dry-run it with a sabotage, and rerun +TP-1 tonight from this branch before it is merged. + +## Change (bench_common.py only) + +- `write_heartbeat`: if the replace is refused (PermissionError), retry every 10 ms, up to 50 times + (0.5 s). If it is still refused, skip this beat and return False instead of raising; the guard's + next beat (0.5 s later) tries again. A long run of skipped beats reads as stale to the scripts, + which then send no move or STOP (unchanged). +- `guard_ok`: if the read is refused (PermissionError), retry every 10 ms, up to 20 times (0.2 s). + If it stays refused, return not OK ("the open guard's heartbeat file stayed locked"). Other + read errors are unchanged ("no heartbeat ..."). +- Nothing else changes: thresholds, the heartbeat's contents, open_guard.py, tp1_run.py. + +## Checks (on this Windows PC, no unit) + +HB1 Stress, two processes for 20 s: a writer calling `write_heartbeat` every 5 ms and a reader + calling `guard_ok` every 1 ms. Head: the writer raises nothing and skips at most 1 % of its + beats; every `guard_ok` call returns OK. + Sabotage: the same run with main's `bench_common.py` (96674ca): the writer raises + PermissionError, or the reader gets at least one not-OK, within the 20 s. +HB2 Replace always refused (os.replace patched to raise PermissionError): `write_heartbeat` + returns False within 0.6 s and raises nothing. + Sabotage: main's version raises PermissionError. +HB3 Read always refused (the read patched to raise PermissionError): `guard_ok` returns not OK, + with "stayed locked", within 0.3 s. + Sabotage: a copy that treats a refused read as OK returns OK, and is flagged. +HB4 The TP-1 dry run still comes out as registered: `tp1_run.py --sim` ends PASS, and + `--sim-guard drop-after-1` ends INCOMPLETE with STOP sent. +D The push gate passes (scan && commit; the PR comes later). + +Then TP-1 reruns on the sealed unit (old carriage, window to 22:00 PT) from this branch's head. From bda6c02d8b0bd57f469911ce4bc3aa588eea135b Mon Sep 17 00:00:00 2001 From: Daniel Frenkel Date: Thu, 12 Nov 2026 20:13:23 -0800 Subject: [PATCH 2/6] Bench: retry the heartbeat file replace and read on Windows Co-Authored-By: Claude Opus 5.5 --- tools/bench/bench_common.py | 27 ++++++++++++++++++++++----- 1 file changed, 22 insertions(+), 5 deletions(-) diff --git a/tools/bench/bench_common.py b/tools/bench/bench_common.py index 21ea162..f746499 100644 --- a/tools/bench/bench_common.py +++ b/tools/bench/bench_common.py @@ -16,6 +16,9 @@ HEARTBEAT = OUT / "guard-heartbeat.json" SITTING = OUT / "sitting.json" MAX_AGE_S = 2.0 +# On Windows a file can't be replaced while another process has it open (and a read can collide +# with the replace): retry briefly (2026-11-bench-heartbeat-retry.md). +RETRY_S, REPLACE_TRIES, READ_TRIES = 0.01, 50, 20 MOVING = ("OPENING", "CLOSING") MOVE_PRESSES = ("Auto-Calibrate", "BENCH: fire scheduled open", "BENCH: fire scheduled close", "BENCH: simulate stall") @@ -23,19 +26,33 @@ # ----------------------------------------------------------------------------- heartbeat def write_heartbeat(connected, watching, carriage, path=HEARTBEAT): + """True once written. If the replace stays refused, this beat is skipped (False); the next one + tries again, and a long run of skipped beats reads as stale to the scripts.""" path.parent.mkdir(parents=True, exist_ok=True) tmp = path.with_suffix(".tmp") tmp.write_text(json.dumps({"t": time.time(), "connected": bool(connected), "watching": bool(watching), "carriage": carriage, "pid": os.getpid()}), encoding="utf-8") - os.replace(tmp, path) + for _ in range(REPLACE_TRIES): + try: + os.replace(tmp, path) + return True + except PermissionError: + time.sleep(RETRY_S) + return False def guard_ok(path=HEARTBEAT, max_age=MAX_AGE_S, now=time.time): """(ok, reason) from the open guard's heartbeat file.""" - try: - hb = json.loads(path.read_text(encoding="utf-8")) - except (OSError, ValueError): - return False, "no heartbeat from the open guard (is open_guard.py running?)" + for _ in range(READ_TRIES): + try: + hb = json.loads(path.read_text(encoding="utf-8")) + break + except PermissionError: + time.sleep(RETRY_S) + except (OSError, ValueError): + return False, "no heartbeat from the open guard (is open_guard.py running?)" + else: + return False, "the open guard's heartbeat file stayed locked" age = now() - hb.get("t", 0) if age > max_age: return False, f"the open guard's heartbeat is {age:.1f} s old" From 59e996313a9e23f2d49b4f9460b54927f4e3f89f Mon Sep 17 00:00:00 2001 From: Daniel Frenkel Date: Thu, 12 Nov 2026 20:14:46 -0800 Subject: [PATCH 3/6] predictions: bench heartbeat retry results on bda6c02 (8/8 as predicted) Co-Authored-By: Claude Opus 5.5 --- .../predictions/2026-11-bench-heartbeat-retry.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tools/regression-gate/predictions/2026-11-bench-heartbeat-retry.md b/tools/regression-gate/predictions/2026-11-bench-heartbeat-retry.md index 3852fd2..a1630b7 100644 --- a/tools/regression-gate/predictions/2026-11-bench-heartbeat-retry.md +++ b/tools/regression-gate/predictions/2026-11-bench-heartbeat-retry.md @@ -45,3 +45,13 @@ HB4 The TP-1 dry run still comes out as registered: `tp1_run.py --sim` ends PASS D The push gate passes (scan && commit; the PR comes later). Then TP-1 reruns on the sealed unit (old carriage, window to 22:00 PT) from this branch's head. + +## Results (2026-11-12, head bda6c02, this Windows PC): 8/8 as predicted + +- HB1 head: the writer made 1932 beats, skipped 0, raised nothing; the reader made 4397 reads, all OK. +- HB1 sabotage (main's bench_common.py): the writer raised PermissionError after 39 beats; the reader + then got not-OK in 10751 of 11944 reads (stale heartbeat). +- HB2 head: returned False after 0.51 s, raised nothing. Sabotage: PermissionError raised at once. +- HB3 head: not OK, "the open guard's heartbeat file stayed locked", after 0.21 s. Sabotage: OK returned. +- HB4: `tp1_run.py --sim` ended PASS; `--sim-guard drop-after-1` ended INCOMPLETE with STOP sent. +- The check script (hb_test.py) is kept with the bench evidence; it goes into the PR with this file. From 0f878e4513b83f533de5471f9b6c4c1dfd0e21d2 Mon Sep 17 00:00:00 2001 From: Daniel Frenkel Date: Thu, 12 Nov 2026 21:02:51 -0800 Subject: [PATCH 4/6] predictions: bench heartbeat Amendment 1, HB5 (a writer that stops still ends a move) Co-Authored-By: Claude Opus 5.5 --- .../predictions/2026-11-bench-heartbeat-retry.md | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/tools/regression-gate/predictions/2026-11-bench-heartbeat-retry.md b/tools/regression-gate/predictions/2026-11-bench-heartbeat-retry.md index a1630b7..5b16479 100644 --- a/tools/regression-gate/predictions/2026-11-bench-heartbeat-retry.md +++ b/tools/regression-gate/predictions/2026-11-bench-heartbeat-retry.md @@ -55,3 +55,17 @@ Then TP-1 reruns on the sealed unit (old carriage, window to 22:00 PT) from this - HB3 head: not OK, "the open guard's heartbeat file stayed locked", after 0.21 s. Sabotage: OK returned. - HB4: `tp1_run.py --sim` ended PASS; `--sim-guard drop-after-1` ended INCOMPLETE with STOP sent. - The check script (hb_test.py) is kept with the bench evidence; it goes into the PR with this file. + +## Amendment 1 — HB5, a writer that stops (review, 2026-11-12; before the check is written) + +- The retries must not hide a guard that has stopped writing. None of HB1-HB4 shows that on the + file path (HB4's drop uses the dry run's own check), so HB5 is added. +- HB5: a writer process using this branch's `write_heartbeat` beats every 0.5 s, then exits + abruptly after 3 s. A MoveGate over a stand-in device that reports State OPENING calls `sleep` + until it raises. Predicted: it sends STOP and raises GuardLost with the "... s old" reason, at + most 2.8 s after the last beat's time stamp (2.0 s age limit, up to 0.5 s between checks, up to + 0.2 s of read retries, 0.1 s slack). + Sabotage: a copy whose `guard_ok` keeps retrying a stale reading for up to 5 s before reporting + it. STOP then comes later than 2.8 s, and HB5 reports it. +- The checks (HB1-HB5) move into `tools/bench/heartbeat_check.py` in this PR, so they can be rerun. +- The full set (HB1-HB5, each with its sabotage, and HB4's dry runs) reruns on the new head. From cca0448689390dc5b228bcdc82d399fa4666a98a Mon Sep 17 00:00:00 2001 From: Daniel Frenkel Date: Thu, 12 Nov 2026 21:03:34 -0800 Subject: [PATCH 5/6] Bench: heartbeat_check.py runs HB1-HB5 with their sabotages Co-Authored-By: Claude Opus 5.5 --- tools/bench/heartbeat_check.py | 192 +++++++++++++++++++++++++++++++++ 1 file changed, 192 insertions(+) create mode 100644 tools/bench/heartbeat_check.py diff --git a/tools/bench/heartbeat_check.py b/tools/bench/heartbeat_check.py new file mode 100644 index 0000000..6b2492c --- /dev/null +++ b/tools/bench/heartbeat_check.py @@ -0,0 +1,192 @@ +#!/usr/bin/env python3 +"""Heartbeat file checks HB1-HB5 — BENCH ONLY (tools/regression-gate/predictions/2026-11-bench-heartbeat-retry.md). + + python tools/bench/heartbeat_check.py + +Runs each check on this folder's bench_common.py and on a patched copy (its sabotage), on this PC, +with no unit. Uses its own temporary heartbeat files, never out/guard-heartbeat.json. Prints, per +check, the registered outcome next to the observed one. +""" +import asyncio, json, os, pathlib, shutil, subprocess, sys, tempfile, time + +HERE = pathlib.Path(__file__).parent +PY = sys.executable +SAB = { # name: [(old, new), ...] applied to a copy of bench_common.py + "no-retry": [ + (" for _ in range(REPLACE_TRIES):\n try:\n os.replace(tmp, path)\n return True\n" + " except PermissionError:\n time.sleep(RETRY_S)\n return False\n", + " os.replace(tmp, path)\n return True\n"), + (" for _ in range(READ_TRIES):\n try:\n hb = json.loads(path.read_text(encoding=\"utf-8\"))\n" + " break\n except PermissionError:\n time.sleep(RETRY_S)\n" + " except (OSError, ValueError):\n return False, \"no heartbeat from the open guard (is open_guard.py running?)\"\n" + " else:\n return False, \"the open guard's heartbeat file stayed locked\"\n", + " try:\n hb = json.loads(path.read_text(encoding=\"utf-8\"))\n" + " except (OSError, ValueError):\n return False, \"no heartbeat from the open guard (is open_guard.py running?)\"\n"), + ], + "refused-read-ok": [ + (" except PermissionError:\n time.sleep(RETRY_S)\n except (OSError, ValueError):", + " except PermissionError:\n return True, \"refused read taken as OK\"\n except (OSError, ValueError):"), + ], + "stale-retry": [ + (" age = now() - hb.get(\"t\", 0)\n", + " age = now() - hb.get(\"t\", 0)\n t_end = time.time() + 5.0\n" + " while age > max_age and time.time() < t_end:\n time.sleep(0.1)\n try:\n" + " hb = json.loads(path.read_text(encoding=\"utf-8\"))\n except (OSError, ValueError):\n" + " pass\n age = now() - hb.get(\"t\", 0)\n"), + ], +} +results = [] + + +def report(name, predicted, ok, detail): + got = "PASS" if ok else "FAIL" + results.append(got == predicted) + print(f"{'as predicted' if got == predicted else 'MISS '} {name}: predicted {predicted}, got {got} ({detail})", + flush=True) + + +def copy_with(sab): + d = tempfile.mkdtemp(prefix=f"hb-{sab}-") + src = (HERE / "bench_common.py").read_text(encoding="utf-8") + for old, new in SAB[sab]: + assert src.count(old) == 1, f"sabotage {sab}: {src.count(old)} matches" + src = src.replace(old, new) + pathlib.Path(d, "bench_common.py").write_text(src, encoding="utf-8") + return d + + +WRITER = r""" +import sys, time, json, os, pathlib +sys.path.insert(0, sys.argv[1]); import bench_common as bc +p, end, period, die = pathlib.Path(sys.argv[2]), time.time() + float(sys.argv[3]), float(sys.argv[4]), sys.argv[5] == "die" +n = skipped = 0; err = None +try: + while time.time() < end: + if bc.write_heartbeat(True, True, "old", path=p) is False: skipped += 1 + n += 1 + time.sleep(period) +except Exception as e: + err = f"{type(e).__name__}: {e}" +print(json.dumps({"beats": n, "skipped": skipped, "error": err}), flush=True) +if die: os._exit(0) +""" +READER = r""" +import sys, time, json, pathlib +sys.path.insert(0, sys.argv[1]); import bench_common as bc +p, end = pathlib.Path(sys.argv[2]), time.time() + float(sys.argv[3]) - 0.3 +n = bad = 0; first = None; err = None +time.sleep(0.3) +try: + while time.time() < end: + ok, why = bc.guard_ok(path=p); n += 1 + if not ok: + bad += 1; first = first or why + time.sleep(0.001) +except Exception as e: + err = f"{type(e).__name__}: {e}" +print(json.dumps({"reads": n, "not_ok": bad, "first": first, "error": err})) +""" + + +def hb1(code_dir, seconds=20.0): + p = os.path.join(tempfile.mkdtemp(prefix="hb1-"), "guard-heartbeat.json") + w = subprocess.Popen([PY, "-c", WRITER, code_dir, p, str(seconds), "0.005", "live"], stdout=subprocess.PIPE, text=True) + r = subprocess.Popen([PY, "-c", READER, code_dir, p, str(seconds)], stdout=subprocess.PIPE, text=True) + wo, ro = json.loads(w.communicate()[0]), json.loads(r.communicate()[0]) + ok = wo["error"] is None and wo["skipped"] <= 0.01 * max(wo["beats"], 1) and ro["error"] is None and ro["not_ok"] == 0 + return ok, f"writer {wo}; reader {ro}" + + +HB2 = r""" +import sys, time, json, pathlib, tempfile +sys.path.insert(0, sys.argv[1]); import bench_common as bc +def refuse(a, b): raise PermissionError(5, "Access is denied") +bc.os.replace = refuse +p = pathlib.Path(tempfile.mkdtemp()) / "guard-heartbeat.json" +t0 = time.time(); err = None; r = None +try: + r = bc.write_heartbeat(True, True, "old", path=p) +except Exception as e: + err = type(e).__name__ +print(json.dumps({"returned": r, "error": err, "s": round(time.time() - t0, 3)})) +""" +HB3 = r""" +import sys, time, json, pathlib, tempfile +sys.path.insert(0, sys.argv[1]); import bench_common as bc +p = pathlib.Path(tempfile.mkdtemp()) / "guard-heartbeat.json"; p.write_text("{}") +def refuse(self, *a, **k): raise PermissionError(5, "Access is denied") +pathlib.Path.read_text = refuse +t0 = time.time(); ok, why = bc.guard_ok(path=p) +print(json.dumps({"ok": ok, "why": why, "s": round(time.time() - t0, 3)})) +""" +HB5 = r""" +import sys, time, json, asyncio, pathlib, subprocess +sys.path.insert(0, sys.argv[1]); import bench_common as bc +p = pathlib.Path(sys.argv[2]) +class Stand: + def __init__(self): self.stop_at = None + def now(self): return time.monotonic() + async def sleep(self, s): await asyncio.sleep(s) + def get(self, name): return "OPENING" if name == "State" else None + async def cover(self, path, cmd, pos=None): + if cmd == "stop" and self.stop_at is None: self.stop_at = time.time() +dev = Stand(); gate = bc.MoveGate(dev, check=lambda: bc.guard_ok(path=p)) +time.sleep(1.0) +why = None +try: + asyncio.run(gate.sleep(30)) +except bc.GuardLost as e: + why = str(e) +last = json.loads(p.read_text(encoding="utf-8"))["t"] +print(json.dumps({"stop_after_last_beat_s": None if dev.stop_at is None else round(dev.stop_at - last, 2), "raised": why})) +""" + + +def run_snippet(code, *args): + return json.loads(subprocess.run([PY, "-c", code, *args], capture_output=True, text=True).stdout) + + +def hb2(code_dir): + o = run_snippet(HB2, code_dir) + return o["error"] is None and o["returned"] is False and o["s"] <= 0.6, str(o) + + +def hb3(code_dir): + o = run_snippet(HB3, code_dir) + return (not o["ok"]) and "stayed locked" in o["why"] and o["s"] <= 0.3, str(o) + + +def hb5(code_dir): + p = os.path.join(tempfile.mkdtemp(prefix="hb5-"), "guard-heartbeat.json") + w = subprocess.Popen([PY, "-c", WRITER, str(HERE), p, "3.0", "0.5", "die"], stdout=subprocess.PIPE, text=True) + o = run_snippet(HB5, code_dir, p) + w.communicate() + s = o["stop_after_last_beat_s"] + return s is not None and s <= 2.8 and o["raised"] is not None and "s old" in o["raised"], str(o) + + +def hb4(): + out = {} + for name, extra, want in (("sim", [], "TP-1: PASS"), ("drop", ["--sim-guard", "drop-after-1"], "TP-1: INCOMPLETE")): + d = tempfile.mkdtemp(prefix=f"hb4-{name}-") + r = subprocess.run([PY, str(HERE / "tp1_run.py"), "--sim", *extra, "--out", d], capture_output=True, text=True, + encoding="utf-8", errors="replace") + line = [l for l in r.stdout.splitlines() if l.startswith("TP-1:")] + stop = "STOP sent" in r.stdout + out[name] = (bool(line) and line[-1].startswith(want) and (name == "sim" or stop), line[-1][:20] if line else None) + return all(v[0] for v in out.values()), str({k: v[1] for k, v in out.items()}) + + +if __name__ == "__main__": + head = str(HERE) + ok, det = hb1(head); report("HB1 head (stress 20 s)", "PASS", ok, det) + ok, det = hb1(copy_with("no-retry")); report("HB1 sabotage: no retries", "FAIL", ok, det) + ok, det = hb2(head); report("HB2 head (replace always refused)", "PASS", ok, det) + ok, det = hb2(copy_with("no-retry")); report("HB2 sabotage: no retries", "FAIL", ok, det) + ok, det = hb3(head); report("HB3 head (read always refused)", "PASS", ok, det) + ok, det = hb3(copy_with("refused-read-ok")); report("HB3 sabotage: refused read taken as OK", "FAIL", ok, det) + ok, det = hb4(); report("HB4 head (TP-1 dry runs)", "PASS", ok, det) + ok, det = hb5(head); report("HB5 head (writer stops mid-move)", "PASS", ok, det) + ok, det = hb5(copy_with("stale-retry")); report("HB5 sabotage: stale reading retried 5 s", "FAIL", ok, det) + print(f"\n{sum(results)}/{len(results)} as predicted") + sys.exit(0 if all(results) else 1) From 5205da6194c071533c9ff02ea44c5b43574ec92b Mon Sep 17 00:00:00 2001 From: Daniel Frenkel Date: Thu, 12 Nov 2026 21:04:44 -0800 Subject: [PATCH 6/6] predictions: bench heartbeat Amendment 1 results on cca0448 (9/9 as predicted) Co-Authored-By: Claude Opus 5.5 --- .../predictions/2026-11-bench-heartbeat-retry.md | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/tools/regression-gate/predictions/2026-11-bench-heartbeat-retry.md b/tools/regression-gate/predictions/2026-11-bench-heartbeat-retry.md index 5b16479..c0e4e97 100644 --- a/tools/regression-gate/predictions/2026-11-bench-heartbeat-retry.md +++ b/tools/regression-gate/predictions/2026-11-bench-heartbeat-retry.md @@ -69,3 +69,14 @@ Then TP-1 reruns on the sealed unit (old carriage, window to 22:00 PT) from this it. STOP then comes later than 2.8 s, and HB5 reports it. - The checks (HB1-HB5) move into `tools/bench/heartbeat_check.py` in this PR, so they can be rerun. - The full set (HB1-HB5, each with its sabotage, and HB4's dry runs) reruns on the new head. + +### Amendment 1 results (2026-11-12, head cca0448, this Windows PC): 9/9 as predicted + +`python tools/bench/heartbeat_check.py`, one run: +- HB1 head: 1995 beats, 0 skipped; 6249 reads, all OK. Sabotage (no retries): PermissionError after + 51 beats. +- HB2 head: False after 0.52 s, nothing raised. Sabotage: PermissionError at once. +- HB3 head: not OK, "stayed locked", after 0.21 s. Sabotage: OK returned. +- HB4: `--sim` PASS; `--sim-guard drop-after-1` INCOMPLETE with STOP sent. +- HB5 head: STOP and GuardLost 2.07 s after the last beat ("heartbeat is 2.1 s old"). Sabotage + (stale reading retried 5 s): STOP 7.11 s after the last beat, reported.