Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
19 commits
Select commit Hold shift + click to select a range
db488c3
gc-plan runtime: data-only packs run by the engine through the CUDA d…
peti12352 Nov 17, 2026
ae5dd8a
gc-plan: decoder backend (PlanBackend, a page backend), one sequence
peti12352 Nov 17, 2026
e4c07d3
gc-plan decoder: several sequences, KV-width ladder, prompt chunks be…
peti12352 Nov 17, 2026
5318abf
index: pack 4 entries (abi_major 3 or 4), gc-plan ranked after gc-c
peti12352 Nov 17, 2026
e596b98
gc-plan: TMA descriptors, init actions, KV width per bucket kind
peti12352 Nov 17, 2026
e6b6e45
gc-plan: view inputs are slices of their base input
peti12352 Nov 17, 2026
76ffd17
gc-plan: affine buckets (a plan at every KV width between two)
peti12352 Nov 17, 2026
763e1a0
engine: prefill-first scheduling and chunk-aligned prompt pieces for …
peti12352 Nov 18, 2026
d9abdbc
gc-plan: Runtime.warm (capture every bucket's graph at once)
peti12352 Nov 18, 2026
1fcbc87
gc-plan: sliding-window cache inputs (attn_inp_{k,v}_idxs_swa, attn_i…
peti12352 Nov 18, 2026
9e330ec
gc-plan: a step is one graph launch (inputs staged in pinned memory, …
peti12352 Nov 18, 2026
64c234c
gc-plan: a live count between the ladder's batch sizes runs in the ne…
peti12352 Nov 18, 2026
be155a2
gc-plan: warm runs every captured graph once (modules loaded before t…
peti12352 Nov 18, 2026
28feb92
gc_plan: the on-device argmax runs 1024 threads per row (gc2's kernel…
peti12352 Nov 18, 2026
37f80fc
gc_plan: GOETHE_PLAN_PDL=0 launches without PDL (profiling)
peti12352 Nov 19, 2026
8276b8d
gc_plan: one scratch arena for all buckets; cuBLAS gemm actions
peti12352 Nov 19, 2026
adca283
gc_plan: JIT on first load: a bucket's near-tie alternative plans tim…
peti12352 Nov 19, 2026
fc29e92
gc_plan: JIT over affine buckets; a short prefill tail runs padded in…
peti12352 Nov 19, 2026
c21912b
gc_plan: a prompt's chunk split by measured launch time
peti12352 Nov 19, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions goethe_engine/engine.py
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,8 @@ def __init__(self, backend, cfg=None, drafter=None, grammar=None, eos_ids=()):
self.cols = np.array([c for k in need for c in at[k]])
self.kv = KVManager(g.num_blocks, g.block_size, cfg.prefix_cache, pages, bounded)
self.sched = Scheduler(cfg, self.kv, drafter, bool(self.width), backend.vocab_size)
self.sched.unmixed = bool(getattr(backend, "batches_all_live", False))
self.sched.align = int(getattr(backend, "prefill_align", 1))
self.inbox, self.live, self.counter = deque(), {}, 0
self.m = dict(steps=0, prompt_tokens=0, generation_tokens=0, prefix_hit_tokens=0, aborted=0, errors=0,
spec_drafted=0, spec_accepted=0, spec_invalid=0)
Expand Down
825 changes: 825 additions & 0 deletions goethe_engine/gc_plan.py

Large diffs are not rendered by default.

11 changes: 6 additions & 5 deletions goethe_engine/index.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@
"structure": [{"variant": str, "sha256": hex, "keys": [str]}], # one per variant in the pack (structure.py)
"files": [{"name": str, "bytes": int, "sha256": hex}]}, # optional, informational: the model files the pack was
# verified on (what a client downloads); never used to match
"arch": "sm_120a", "abi_major": 3, "runtime": {"kind": str, "version": int}, "tier": str,
"arch": "sm_120a", "abi_major": 3 | 4, "runtime": {"kind": str, "version": int}, "tier": str, # 4: blobs from the store
"file": "<model>.gcpack", # the asset name (ASCII slug), one file per model
"manifest_sha256": hex, # sha256 of the pack's manifest bytes: what `sig` signs (with the
"sig": {gc-sig-1 object}, # domain prefix of pack.py) and what a range fetch is checked against
Expand All @@ -26,7 +26,8 @@
SCHEMA, VERSION = "gc-pack-index", 3
HEX = re.compile(r"^[0-9a-f]{64}$")
SLUG = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]{0,127}\.gcpack$")
RT_PREF = {"gc-c": 0} # when several entries match: ranked by runtime, then tier, then manifest hash
RT_PREF = {"gc-c": 0, "gc-plan": 1} # when several entries match: ranked by runtime, then tier, then manifest hash
MAJORS = (3, 4) # the pack majors this loader reads (pack.py: the container's magic fixes it)


class IndexError_(Exception):
Expand Down Expand Up @@ -65,7 +66,7 @@ def validate_entry(e, i):
and isinstance(x.get("bytes"), int) and x["bytes"] >= 0 and HEX.match(str(x.get("sha256")))
for x in fl)), f"{w}: model.files is a list of {{name, bytes, sha256}}")
need(isinstance(e.get("arch"), str) and re.fullmatch(r"sm_\d+[a-z]?", e["arch"]), f"{w}: arch is not sm_XX")
need(e.get("abi_major") == 3, f"{w}: abi_major")
need(e.get("abi_major") in MAJORS, f"{w}: abi_major")
r = e.get("runtime")
need(isinstance(r, dict) and isinstance(r.get("kind"), str) and isinstance(r.get("version"), int), f"{w}: runtime")
need(isinstance(e.get("tier"), str) and re.fullmatch(r"[a-z0-9_-]+", e["tier"]), f"{w}: tier")
Expand Down Expand Up @@ -118,10 +119,10 @@ def model_sets(model_dir):
def match_dir(ix, model_dir, arch, engine_version=__version__, tier=None):
"""The entries for this arch, ABI, runtime and engine version one of whose variant structures a file set of model_dir has
(header-only), best first (see RT_PREF); each carries "variant": the matching variant id."""
ev, la = vtuple(engine_version, "engine version"), pack.semver(pack.ABI)[0]
ev = vtuple(engine_version, "engine version")
memo, sets, hit = {}, model_sets(model_dir), []
for e in ix["entries"]:
if not (e["arch"] == arch and e["abi_major"] == la and e["runtime"]["version"] in pack.RUNTIMES.get(e["runtime"]["kind"], ())
if not (e["arch"] == arch and e["abi_major"] in MAJORS and e["runtime"]["version"] in pack.RUNTIMES.get(e["runtime"]["kind"], ())
and vtuple(e["min_engine"], "min_engine") <= ev and (tier is None or e["tier"] == tier)):
continue
found = None
Expand Down
2 changes: 1 addition & 1 deletion goethe_engine/pack.py
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@
ABI4, MAGIC4, DOMAIN4 = "4.0.0", b"GCPACK4\n", b"gc-pack-4\n" # pack 4: the same framing, blobs by hash from a store
MAX_HEADER = 64 << 20 # bytes of container header JSON
SCHEMA = "gc-pack"
RUNTIMES = {"gc-c": (1,)} # runtime kinds this loader runs, with their versions
RUNTIMES = {"gc-c": (1,), "gc-plan": (1,)} # runtime kinds this loader runs, with their versions (gc-plan: gc_plan.py)
MAX_RAW = 4 << 30 # bytes: no blob inflates past this (xz bombs are refused as the output grows)
XZ_MEM = 64 << 20 # bytes: xz decoder memory limit
MAX_JSON = 16 << 20 # bytes: larger manifests are refused before parsing
Expand Down
8 changes: 8 additions & 0 deletions goethe_engine/registry.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,14 @@


def _gc_c(pack, weights, *, mesh, model_type="decoder", **kw):
m = _manifest(pack)
if isinstance(m, dict) and (m.get("runtime") or {}).get("kind") == "gc-plan": # a data-only pack: the engine runs its plans
from .gc_plan import PlanBackend, PlanEmbed
if model_type == "embedding":
return PlanEmbed(pack, weights, device=mesh.devices[0], **kw)
if model_type == "decoder":
return PlanBackend(pack, weights, device=mesh.devices[0], **kw)
raise P.PackError(f"gc-plan packs serve embedding and decoder models so far, not {model_type}")
if model_type == "embedding":
from .embed import GcEmbed
return GcEmbed(pack, weights, device=mesh.devices[0], **kw)
Expand Down
28 changes: 26 additions & 2 deletions goethe_engine/scheduler.py
Original file line number Diff line number Diff line change
Expand Up @@ -82,6 +82,8 @@ def __init__(self, cfg, kv: KVManager, drafter=None, spec_round=False, vocab=1 <
self.cfg, self.kv, self.drafter, self.spec_round, self.vocab = cfg, kv, drafter, spec_round, vocab
self.waiting, self.running = [], []
self.preemptions = 0
self.unmixed = False # the backend batches decodes only when every live sequence decodes (gc-plan buckets): prefill first
self.align = 1 # the backend's prefill chunk: a prompt piece short of the prompt's end is a whole number of them

def add(self, seq):
self.waiting.append(seq)
Expand Down Expand Up @@ -115,7 +117,15 @@ def _chain(self, tokens, npages):
return out

def schedule(self):
"""(items, copies): the work of one step and the CoW block copies to run before it."""
"""(items, copies): the work of one step and the CoW block copies to run before it. unmixed: a step that can carry
prefill work carries no decodes (they wait one step and then run batched); it never leaves a step empty."""
if self.unmixed:
items, copies = self._schedule(skip_decode=True)
if items:
return items, copies
return self._schedule()

def _schedule(self, skip_decode=False):
cfg, kv = self.cfg, self.kv
budget, items, copies, preempted = cfg.max_num_batched_tokens, [], [], False
self.preempted = []
Expand All @@ -135,13 +145,21 @@ def schedule(self):
while i < len(self.running) and budget > 0:
s = self.running[i]
need = chunk(s, len(s.tokens) - s.computed, kv.snaps or kv.bsnaps)
if skip_decode and len(s.tokens) - s.computed == 1:
i += 1
continue
t = trees.get(s.sid, empty)
if need + len(t) > budget:
if len(t) and strict(s):
i += 1
continue
t = t.cut(budget - need)
n = min(need + len(t), budget)
if n < need and not len(t) and self.align > 1:
n -= n % self.align
if n <= 0:
i += 1
continue
while (pairs := kv.reserve(s, s.computed + n)) is None:
if len(t) and not strict(s):
t, n = empty, min(need, budget)
Expand Down Expand Up @@ -174,7 +192,13 @@ def schedule(self):
if lead * kv.bs >= 32:
continue
kv.attach(s)
n = min(chunk(s, len(s.tokens) - s.computed, kv.snaps or kv.bsnaps), budget)
need = chunk(s, len(s.tokens) - s.computed, kv.snaps or kv.bsnaps)
n = min(need, budget)
if n < need and self.align > 1:
n -= n % self.align
if n <= 0:
kv.release(s)
break
if len(s.tokens) - s.computed == 1 and strict(s) and s.nout:
n = 0 # the anchor alone: next step's decode carries its drafts
pairs = kv.reserve(s, s.computed + n)
Expand Down
26 changes: 26 additions & 0 deletions goethe_engine/structure.py
Original file line number Diff line number Diff line change
Expand Up @@ -87,6 +87,32 @@ def gguf_header(path):
return kv, tens


def gguf_offsets(path):
"""{tensor name: absolute file offset of its data} of a GGUF file (general.alignment, default 32): where a runtime reads a
tensor's bytes; sizes come from what reads them."""
with open(path, "rb") as f:
r = _Rd(f)
if r.take(4) != b"GGUF" or r.get("I") not in (2, 3):
raise StructureError(f"{path}: not a GGUF v2/v3 file")
nt, nkv = r.get("Q"), r.get("Q")
kv = {}
for _ in range(nkv):
k = r.str()
kv[k] = _value(r, r.get("I"))
rel = {}
for _ in range(nt):
name, nd = r.str(), r.get("I")
if nd > 8:
raise StructureError(f"{path}: tensor rank")
for _ in range(nd):
r.get("Q")
r.get("I")
rel[name] = r.get("Q")
al = int(kv.get("general.alignment", 32))
start = -(-f.tell() // al) * al
return {n: start + o for n, o in rel.items()}


def st_header(path):
"""[(name, shape, dtype)] of a safetensors file: header only."""
with open(path, "rb") as f:
Expand Down
116 changes: 116 additions & 0 deletions tests/test_gc_plan.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,116 @@
"""gc-plan (data-only packs run by gc_plan.py): GGUF data offsets, the inputs the engine generates, the runtime kind accepted. CPU only;
the device side (Runtime, PlanEmbed) is checked on a GPU against llama.cpp when a pack is built."""
import json

import numpy as np

from goethe_engine import gc_plan as G
from goethe_engine import pack as P
from goethe_engine.structure import gguf_offsets
from toypack import gguf


def test_gguf_offsets(tmp_path):
w = tmp_path / "w.gguf"
gguf(str(w), [("a.weight", [64, 8], 2), ("b.weight", [64], 0)], align=32)
off = gguf_offsets(str(w))
raw = w.read_bytes()
assert set(off) == {"a.weight", "b.weight"} and off["b.weight"] - off["a.weight"] == 64
assert off["a.weight"] % 32 == 0 and off["b.weight"] + 64 == len(raw) # the data section ends the file


def test_encoder_mask_pads_keys_and_cells():
T, n_real = 128, 100 # 100 real tokens in a 128 chunk; keys padded to 256 cells (f16)
hp = {"causal": False, "n_swa": 0, "masks": {"self_kq_mask": {"esize": 2, "swa": False}}}
st = G.Step([7] * T, [0] * T, range(T), range(T), [0] * T, T, T, range(T), n_real=n_real)
m = np.frombuffer(G.generate("CUDA0#self_kq_mask#0", 2 * T * 256, st, hp), np.uint16).reshape(T, 256)
assert (m[:, :n_real] == 0).all() and (m[:, n_real:] == G.NEG_INF_F16).all() # pad queries see the real keys: finite rows


def test_swa_mask_is_symmetric():
T = 64
hp = {"causal": False, "n_swa": 16, "masks": {"leaf_3": {"esize": 4, "swa": True}}}
st = G.Step([1] * T, [0] * T, range(T), range(T), [0] * T, T, T, range(T))
m = np.frombuffer(G.generate("CUDA0#leaf_3#0", 4 * T * T, st, hp), np.float32).reshape(T, T)
q, k = np.indices((T, T))
assert ((m == 0) == (np.abs(q - k) <= 8)).all()


def test_pooling_and_fills():
st = G.Step([3] * 8, [0] * 8, range(8), range(8), [0] * 8, 8, 8, range(8), n_real=5)
mean = np.frombuffer(G.generate("CUDA0#mean#0", 32, st, {}), np.float32)
assert np.allclose(mean[:5], 0.2) and (mean[5:] == 0).all()
assert np.frombuffer(G.generate("CUDA0#cls#0", 4, st, {}), np.int32).tolist() == [0]
rows = np.frombuffer(G.generate("CUDA0#inp_scale_rows#0", 32, st, {"fills": {"inp_scale_rows": 2.5}}), np.float32)
assert (rows == 2.5).all()


def test_view_input_is_a_slice_of_its_base():
st = G.Step([5] * 4, [0, 1, 2, 3], range(4), range(4), [0] * 4, 4, 4, range(4)) # four sequences, one token each
hp = {"views": {"CUDA0#rs_s_copy (view)#0@12": 1}}
v = np.frombuffer(G.generate("CUDA0#rs_s_copy (view)#0", 12, st, hp), np.int32)
assert v.tolist() == [1, 2, 3]


def test_lerp_plan_interpolates_widths():
import struct
fake = lambda v: struct.pack("<Q", G.FAKE0 + v).hex()
fd = lambda d: struct.pack("<3I", *G._fastdiv(d)).hex()
host = struct.pack("<Q", 0x7FFF00001000).hex()
act = lambda w, h: {"launch": "k", "g": [w // 256, 1, 1], "args": [fake(4 * w) + host[:0], fd(w // 128), struct.pack("<i", w).hex(), h],
"ptrs": [[0, 0, 4 * w]]}
p0 = {"io": {"n_kv": 512}, "actions": [act(512, host)], "buffers": [{"role": "compute", "size": 1024 * 512}]}
p1 = {"io": {"n_kv": 2048}, "actions": [act(2048, struct.pack("<Q", 0x7FFD00002000).hex())],
"buffers": [{"role": "compute", "size": 1024 * 2048}]}
p = G.lerp_plan(p0, p1, (1536 - 512, 2048 - 512))
want = act(1536, host)
assert p["io"]["n_kv"] == 1536 and p["buffers"][0]["size"] == 1024 * 1536 and p["actions"][0] == want
try:
G.lerp_plan({"io": {"n_kv": 1}}, {"io": {"n_kv": 2}}, (1, 3))
except P.PackError:
pass
else:
raise AssertionError("a non-integer step must be refused")


def test_unknown_input_refused():
st = G.Step([0], [0], [0], [0], [0], 1, 1, [0])
try:
G.generate("CUDA0#mystery#0", 4, st, {})
except P.PackError as e:
assert "mystery" in str(e)
else:
raise AssertionError("an input of no known kind must be refused")


def test_runtime_kind_accepted():
assert P.RUNTIMES["gc-plan"] == (1,)
m = {"runtime": {"kind": "gc-plan", "version": 1}}
assert json.loads(json.dumps(m))["runtime"]["kind"] in P.RUNTIMES


def _ring_reference(pos_by_stream, S, W, K):
"""a sliding-window ring simulated: every position up to the stream's last is written to cell p % S (newest wins); a query
at p sees cell j iff it holds p_j with p - W < p_j <= p (llama.cpp's LLAMA_SWA_TYPE_STANDARD)"""
rows = []
for ps in pos_by_stream:
held = {}
for q in range(max(ps) + 1):
held[q % S] = q
for p in ps:
rows.append([j in held and held[j] <= p and p - held[j] < W for j in range(K)])
return np.array(rows)


def test_swa_ring_inputs_match_a_simulated_cache():
S, W, hp = 1536, 1024, {"swa_cells": 1536, "n_swa": 1024}
cases = [([4096] * 4, 1536), ([2047], 1536), (list(range(3584, 4096)), 1536), (list(range(0, 300)), 512), ([5], 256)]
for ps, K in cases:
multi = len(ps) > 1 and len(set(ps)) == 1 # a decode of several sequences: one token per stream
streams = list(range(len(ps))) if multi else [0] * len(ps)
st = G.Step([1] * len(ps), streams, ps, ps, streams, 4352, 0, range(len(ps)))
m = np.frombuffer(G.generate("CUDA0#attn_inp_kq_mask_swa#0", 2 * len(ps) * K, st, hp), np.uint16).reshape(len(ps), K)
ref = _ring_reference([[p] for p in ps] if multi else [ps], S, W, K)
assert ((m == 0) == ref).all(), (ps[:2], K)
idx = np.frombuffer(G.generate("CUDA0#attn_inp_k_idxs_swa#0", 8 * len(ps), st, hp), np.int64)
assert (idx == np.array(streams) * S + np.array(ps) % S).all()
11 changes: 11 additions & 0 deletions tests/test_index.py
Original file line number Diff line number Diff line change
Expand Up @@ -69,6 +69,17 @@ def test_a_finetune_matches_and_changed_structure_does_not(key, tmp_path):
assert I.match_dir(ix, str(tmp_path / "empty"), "sm_120a", "0.2.0") == []


def test_pack4_gc_plan_entries_match_after_gc_c(key, tmp_path):
ref = model_dir(tmp_path / "ref", TARGET)
raw, sig = signed(key, index(entries=[
entry(variants=(("v0", sha(ref)),), abi_major=4, runtime=dict(kind="gc-plan", version=1), manifest_sha256=H("c")),
entry(variants=(("q4", sha(ref)),))]))
ix, _ = I.verify(raw, sig, 5, NOW, (key.kid,))
assert [(e["runtime"]["kind"], e["abi_major"]) for e in I.match_dir(ix, ref, "sm_120a", "0.2.0")] == [("gc-c", 3), ("gc-plan", 4)]
with pytest.raises(I.IndexError_, match="abi_major"):
I.verify(*signed(key, index(entries=[entry(abi_major=5)])), 5, NOW, (key.kid,))


def test_rollback_and_freeze_refused(key):
raw, sig = signed(key, index(serial=4))
with pytest.raises(I.IndexError_, match="rollback"):
Expand Down
2 changes: 1 addition & 1 deletion tests/test_pack.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@


def test_loader_is_the_pinned_abi():
assert P.ABI == "3.0.0" and P.MAGIC == b"GCPACK3\n" and P.SCHEMA == "gc-pack" and list(P.RUNTIMES) == ["gc-c"]
assert P.ABI == "3.0.0" and P.MAGIC == b"GCPACK3\n" and P.SCHEMA == "gc-pack" and list(P.RUNTIMES) == ["gc-c", "gc-plan"]


def test_caps_and_numerics_of_a_bucket(toy, key):
Expand Down