Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
27 changes: 22 additions & 5 deletions tools/bench/bench_common.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,26 +16,43 @@
HEARTBEAT = OUT / "guard-heartbeat.json"
SITTING = OUT / "sitting.json"
MAX_AGE_S = 2.0
# On Windows a file can't be replaced while another process has it open (and a read can collide
# with the replace): retry briefly (2026-11-bench-heartbeat-retry.md).
RETRY_S, REPLACE_TRIES, READ_TRIES = 0.01, 50, 20
MOVING = ("OPENING", "CLOSING")
MOVE_PRESSES = ("Auto-Calibrate", "BENCH: fire scheduled open", "BENCH: fire scheduled close",
"BENCH: simulate stall")


# ----------------------------------------------------------------------------- heartbeat
def write_heartbeat(connected, watching, carriage, path=HEARTBEAT):
"""True once written. If the replace stays refused, this beat is skipped (False); the next one
tries again, and a long run of skipped beats reads as stale to the scripts."""
path.parent.mkdir(parents=True, exist_ok=True)
tmp = path.with_suffix(".tmp")
tmp.write_text(json.dumps({"t": time.time(), "connected": bool(connected), "watching": bool(watching),
"carriage": carriage, "pid": os.getpid()}), encoding="utf-8")
os.replace(tmp, path)
for _ in range(REPLACE_TRIES):
try:
os.replace(tmp, path)
return True
except PermissionError:
time.sleep(RETRY_S)
return False


def guard_ok(path=HEARTBEAT, max_age=MAX_AGE_S, now=time.time):
"""(ok, reason) from the open guard's heartbeat file."""
try:
hb = json.loads(path.read_text(encoding="utf-8"))
except (OSError, ValueError):
return False, "no heartbeat from the open guard (is open_guard.py running?)"
for _ in range(READ_TRIES):
try:
hb = json.loads(path.read_text(encoding="utf-8"))
break
except PermissionError:
time.sleep(RETRY_S)
except (OSError, ValueError):
return False, "no heartbeat from the open guard (is open_guard.py running?)"
else:
return False, "the open guard's heartbeat file stayed locked"
age = now() - hb.get("t", 0)
if age > max_age:
return False, f"the open guard's heartbeat is {age:.1f} s old"
Expand Down
192 changes: 192 additions & 0 deletions tools/bench/heartbeat_check.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,192 @@
#!/usr/bin/env python3
"""Heartbeat file checks HB1-HB5 — BENCH ONLY (tools/regression-gate/predictions/2026-11-bench-heartbeat-retry.md).

python tools/bench/heartbeat_check.py

Runs each check on this folder's bench_common.py and on a patched copy (its sabotage), on this PC,
with no unit. Uses its own temporary heartbeat files, never out/guard-heartbeat.json. Prints, per
check, the registered outcome next to the observed one.
"""
import asyncio, json, os, pathlib, shutil, subprocess, sys, tempfile, time

HERE = pathlib.Path(__file__).parent
PY = sys.executable
SAB = { # name: [(old, new), ...] applied to a copy of bench_common.py
"no-retry": [
(" for _ in range(REPLACE_TRIES):\n try:\n os.replace(tmp, path)\n return True\n"
" except PermissionError:\n time.sleep(RETRY_S)\n return False\n",
" os.replace(tmp, path)\n return True\n"),
(" for _ in range(READ_TRIES):\n try:\n hb = json.loads(path.read_text(encoding=\"utf-8\"))\n"
" break\n except PermissionError:\n time.sleep(RETRY_S)\n"
" except (OSError, ValueError):\n return False, \"no heartbeat from the open guard (is open_guard.py running?)\"\n"
" else:\n return False, \"the open guard's heartbeat file stayed locked\"\n",
" try:\n hb = json.loads(path.read_text(encoding=\"utf-8\"))\n"
" except (OSError, ValueError):\n return False, \"no heartbeat from the open guard (is open_guard.py running?)\"\n"),
],
"refused-read-ok": [
(" except PermissionError:\n time.sleep(RETRY_S)\n except (OSError, ValueError):",
" except PermissionError:\n return True, \"refused read taken as OK\"\n except (OSError, ValueError):"),
],
"stale-retry": [
(" age = now() - hb.get(\"t\", 0)\n",
" age = now() - hb.get(\"t\", 0)\n t_end = time.time() + 5.0\n"
" while age > max_age and time.time() < t_end:\n time.sleep(0.1)\n try:\n"
" hb = json.loads(path.read_text(encoding=\"utf-8\"))\n except (OSError, ValueError):\n"
" pass\n age = now() - hb.get(\"t\", 0)\n"),
],
}
results = []


def report(name, predicted, ok, detail):
got = "PASS" if ok else "FAIL"
results.append(got == predicted)
print(f"{'as predicted' if got == predicted else 'MISS '} {name}: predicted {predicted}, got {got} ({detail})",
flush=True)


def copy_with(sab):
d = tempfile.mkdtemp(prefix=f"hb-{sab}-")
src = (HERE / "bench_common.py").read_text(encoding="utf-8")
for old, new in SAB[sab]:
assert src.count(old) == 1, f"sabotage {sab}: {src.count(old)} matches"
src = src.replace(old, new)
pathlib.Path(d, "bench_common.py").write_text(src, encoding="utf-8")
return d


WRITER = r"""
import sys, time, json, os, pathlib
sys.path.insert(0, sys.argv[1]); import bench_common as bc
p, end, period, die = pathlib.Path(sys.argv[2]), time.time() + float(sys.argv[3]), float(sys.argv[4]), sys.argv[5] == "die"
n = skipped = 0; err = None
try:
while time.time() < end:
if bc.write_heartbeat(True, True, "old", path=p) is False: skipped += 1
n += 1
time.sleep(period)
except Exception as e:
err = f"{type(e).__name__}: {e}"
print(json.dumps({"beats": n, "skipped": skipped, "error": err}), flush=True)
if die: os._exit(0)
"""
READER = r"""
import sys, time, json, pathlib
sys.path.insert(0, sys.argv[1]); import bench_common as bc
p, end = pathlib.Path(sys.argv[2]), time.time() + float(sys.argv[3]) - 0.3
n = bad = 0; first = None; err = None
time.sleep(0.3)
try:
while time.time() < end:
ok, why = bc.guard_ok(path=p); n += 1
if not ok:
bad += 1; first = first or why
time.sleep(0.001)
except Exception as e:
err = f"{type(e).__name__}: {e}"
print(json.dumps({"reads": n, "not_ok": bad, "first": first, "error": err}))
"""


def hb1(code_dir, seconds=20.0):
p = os.path.join(tempfile.mkdtemp(prefix="hb1-"), "guard-heartbeat.json")
w = subprocess.Popen([PY, "-c", WRITER, code_dir, p, str(seconds), "0.005", "live"], stdout=subprocess.PIPE, text=True)
r = subprocess.Popen([PY, "-c", READER, code_dir, p, str(seconds)], stdout=subprocess.PIPE, text=True)
wo, ro = json.loads(w.communicate()[0]), json.loads(r.communicate()[0])
ok = wo["error"] is None and wo["skipped"] <= 0.01 * max(wo["beats"], 1) and ro["error"] is None and ro["not_ok"] == 0
return ok, f"writer {wo}; reader {ro}"


HB2 = r"""
import sys, time, json, pathlib, tempfile
sys.path.insert(0, sys.argv[1]); import bench_common as bc
def refuse(a, b): raise PermissionError(5, "Access is denied")
bc.os.replace = refuse
p = pathlib.Path(tempfile.mkdtemp()) / "guard-heartbeat.json"
t0 = time.time(); err = None; r = None
try:
r = bc.write_heartbeat(True, True, "old", path=p)
except Exception as e:
err = type(e).__name__
print(json.dumps({"returned": r, "error": err, "s": round(time.time() - t0, 3)}))
"""
HB3 = r"""
import sys, time, json, pathlib, tempfile
sys.path.insert(0, sys.argv[1]); import bench_common as bc
p = pathlib.Path(tempfile.mkdtemp()) / "guard-heartbeat.json"; p.write_text("{}")
def refuse(self, *a, **k): raise PermissionError(5, "Access is denied")
pathlib.Path.read_text = refuse
t0 = time.time(); ok, why = bc.guard_ok(path=p)
print(json.dumps({"ok": ok, "why": why, "s": round(time.time() - t0, 3)}))
"""
HB5 = r"""
import sys, time, json, asyncio, pathlib, subprocess
sys.path.insert(0, sys.argv[1]); import bench_common as bc
p = pathlib.Path(sys.argv[2])
class Stand:
def __init__(self): self.stop_at = None
def now(self): return time.monotonic()
async def sleep(self, s): await asyncio.sleep(s)
def get(self, name): return "OPENING" if name == "State" else None
async def cover(self, path, cmd, pos=None):
if cmd == "stop" and self.stop_at is None: self.stop_at = time.time()
dev = Stand(); gate = bc.MoveGate(dev, check=lambda: bc.guard_ok(path=p))
time.sleep(1.0)
why = None
try:
asyncio.run(gate.sleep(30))
except bc.GuardLost as e:
why = str(e)
last = json.loads(p.read_text(encoding="utf-8"))["t"]
print(json.dumps({"stop_after_last_beat_s": None if dev.stop_at is None else round(dev.stop_at - last, 2), "raised": why}))
"""


def run_snippet(code, *args):
return json.loads(subprocess.run([PY, "-c", code, *args], capture_output=True, text=True).stdout)


def hb2(code_dir):
o = run_snippet(HB2, code_dir)
return o["error"] is None and o["returned"] is False and o["s"] <= 0.6, str(o)


def hb3(code_dir):
o = run_snippet(HB3, code_dir)
return (not o["ok"]) and "stayed locked" in o["why"] and o["s"] <= 0.3, str(o)


def hb5(code_dir):
p = os.path.join(tempfile.mkdtemp(prefix="hb5-"), "guard-heartbeat.json")
w = subprocess.Popen([PY, "-c", WRITER, str(HERE), p, "3.0", "0.5", "die"], stdout=subprocess.PIPE, text=True)
o = run_snippet(HB5, code_dir, p)
w.communicate()
s = o["stop_after_last_beat_s"]
return s is not None and s <= 2.8 and o["raised"] is not None and "s old" in o["raised"], str(o)


def hb4():
out = {}
for name, extra, want in (("sim", [], "TP-1: PASS"), ("drop", ["--sim-guard", "drop-after-1"], "TP-1: INCOMPLETE")):
d = tempfile.mkdtemp(prefix=f"hb4-{name}-")
r = subprocess.run([PY, str(HERE / "tp1_run.py"), "--sim", *extra, "--out", d], capture_output=True, text=True,
encoding="utf-8", errors="replace")
line = [l for l in r.stdout.splitlines() if l.startswith("TP-1:")]
stop = "STOP sent" in r.stdout
out[name] = (bool(line) and line[-1].startswith(want) and (name == "sim" or stop), line[-1][:20] if line else None)
return all(v[0] for v in out.values()), str({k: v[1] for k, v in out.items()})


if __name__ == "__main__":
head = str(HERE)
ok, det = hb1(head); report("HB1 head (stress 20 s)", "PASS", ok, det)
ok, det = hb1(copy_with("no-retry")); report("HB1 sabotage: no retries", "FAIL", ok, det)
ok, det = hb2(head); report("HB2 head (replace always refused)", "PASS", ok, det)
ok, det = hb2(copy_with("no-retry")); report("HB2 sabotage: no retries", "FAIL", ok, det)
ok, det = hb3(head); report("HB3 head (read always refused)", "PASS", ok, det)
ok, det = hb3(copy_with("refused-read-ok")); report("HB3 sabotage: refused read taken as OK", "FAIL", ok, det)
ok, det = hb4(); report("HB4 head (TP-1 dry runs)", "PASS", ok, det)
ok, det = hb5(head); report("HB5 head (writer stops mid-move)", "PASS", ok, det)
ok, det = hb5(copy_with("stale-retry")); report("HB5 sabotage: stale reading retried 5 s", "FAIL", ok, det)
print(f"\n{sum(results)}/{len(results)} as predicted")
sys.exit(0 if all(results) else 1)
82 changes: 82 additions & 0 deletions tools/regression-gate/predictions/2026-11-bench-heartbeat-retry.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,82 @@
# Bench tools: open guard heartbeat file on Windows — predictions

Written and committed before the code change, on branch `bench/heartbeat-retry` (base: origin/main
`96674ca`). No check below had been run when this file was committed.

## What happened (TP-1 sitting, 2026-11-12, 20:02 PT, sealed unit, old carriage)

About 70 s after open_guard.py started, and 10 s into tp1_run.py, open_guard.py exited with
`PermissionError: [WinError 5] Access is denied` from `os.replace(guard-heartbeat.tmp,
guard-heartbeat.json)` in `bench_common.write_heartbeat`. On Windows, replacing a file fails while
another process has it open, and tp1_run.py reads that file before and during every move. With the
guard gone, its heartbeat went stale; tp1_run.py sent STOP mid-move (the 3.6 s stale reading) and
ended INCOMPLETE. The interlock worked; the run gave no TP-1 reading (instrument failure, disclosed).
The reader has the same exposure: `guard_ok` turns a read that collides with a replace into "no
heartbeat", which would also stop a move.
Daniel (2026-11-12, question tool): fix now, register it, dry-run it with a sabotage, and rerun
TP-1 tonight from this branch before it is merged.

## Change (bench_common.py only)

- `write_heartbeat`: if the replace is refused (PermissionError), retry every 10 ms, up to 50 times
(0.5 s). If it is still refused, skip this beat and return False instead of raising; the guard's
next beat (0.5 s later) tries again. A long run of skipped beats reads as stale to the scripts,
which then send no move or STOP (unchanged).
- `guard_ok`: if the read is refused (PermissionError), retry every 10 ms, up to 20 times (0.2 s).
If it stays refused, return not OK ("the open guard's heartbeat file stayed locked"). Other
read errors are unchanged ("no heartbeat ...").
- Nothing else changes: thresholds, the heartbeat's contents, open_guard.py, tp1_run.py.

## Checks (on this Windows PC, no unit)

HB1 Stress, two processes for 20 s: a writer calling `write_heartbeat` every 5 ms and a reader
calling `guard_ok` every 1 ms. Head: the writer raises nothing and skips at most 1 % of its
beats; every `guard_ok` call returns OK.
Sabotage: the same run with main's `bench_common.py` (96674ca): the writer raises
PermissionError, or the reader gets at least one not-OK, within the 20 s.
HB2 Replace always refused (os.replace patched to raise PermissionError): `write_heartbeat`
returns False within 0.6 s and raises nothing.
Sabotage: main's version raises PermissionError.
HB3 Read always refused (the read patched to raise PermissionError): `guard_ok` returns not OK,
with "stayed locked", within 0.3 s.
Sabotage: a copy that treats a refused read as OK returns OK, and is flagged.
HB4 The TP-1 dry run still comes out as registered: `tp1_run.py --sim` ends PASS, and
`--sim-guard drop-after-1` ends INCOMPLETE with STOP sent.
D The push gate passes (scan && commit; the PR comes later).

Then TP-1 reruns on the sealed unit (old carriage, window to 22:00 PT) from this branch's head.

## Results (2026-11-12, head bda6c02, this Windows PC): 8/8 as predicted

- HB1 head: the writer made 1932 beats, skipped 0, raised nothing; the reader made 4397 reads, all OK.
- HB1 sabotage (main's bench_common.py): the writer raised PermissionError after 39 beats; the reader
then got not-OK in 10751 of 11944 reads (stale heartbeat).
- HB2 head: returned False after 0.51 s, raised nothing. Sabotage: PermissionError raised at once.
- HB3 head: not OK, "the open guard's heartbeat file stayed locked", after 0.21 s. Sabotage: OK returned.
- HB4: `tp1_run.py --sim` ended PASS; `--sim-guard drop-after-1` ended INCOMPLETE with STOP sent.
- The check script (hb_test.py) is kept with the bench evidence; it goes into the PR with this file.

## Amendment 1 — HB5, a writer that stops (review, 2026-11-12; before the check is written)

- The retries must not hide a guard that has stopped writing. None of HB1-HB4 shows that on the
file path (HB4's drop uses the dry run's own check), so HB5 is added.
- HB5: a writer process using this branch's `write_heartbeat` beats every 0.5 s, then exits
abruptly after 3 s. A MoveGate over a stand-in device that reports State OPENING calls `sleep`
until it raises. Predicted: it sends STOP and raises GuardLost with the "... s old" reason, at
most 2.8 s after the last beat's time stamp (2.0 s age limit, up to 0.5 s between checks, up to
0.2 s of read retries, 0.1 s slack).
Sabotage: a copy whose `guard_ok` keeps retrying a stale reading for up to 5 s before reporting
it. STOP then comes later than 2.8 s, and HB5 reports it.
- The checks (HB1-HB5) move into `tools/bench/heartbeat_check.py` in this PR, so they can be rerun.
- The full set (HB1-HB5, each with its sabotage, and HB4's dry runs) reruns on the new head.

### Amendment 1 results (2026-11-12, head cca0448, this Windows PC): 9/9 as predicted

`python tools/bench/heartbeat_check.py`, one run:
- HB1 head: 1995 beats, 0 skipped; 6249 reads, all OK. Sabotage (no retries): PermissionError after
51 beats.
- HB2 head: False after 0.52 s, nothing raised. Sabotage: PermissionError at once.
- HB3 head: not OK, "stayed locked", after 0.21 s. Sabotage: OK returned.
- HB4: `--sim` PASS; `--sim-guard drop-after-1` INCOMPLETE with STOP sent.
- HB5 head: STOP and GuardLost 2.07 s after the last beat ("heartbeat is 2.1 s old"). Sabotage
(stale reading retried 5 s): STOP 7.11 s after the last beat, reported.
Loading