From c349fe622e34a05887c34e34e918b76b1fba7c04 Mon Sep 17 00:00:00 2001 From: Omkar Tripathi <38633467+triggeredcode@users.noreply.github.com> Date: Tue, 20 Oct 2026 11:06:35 +0530 Subject: [PATCH] feat(evals): add routing and context experiment baseline (#51) --- docs/evals.md | 11 + package.json | 1 + packages/evals/scripts/experiment.mjs | 25 ++ packages/evals/src/experiment/baseline.ts | 85 ++++ packages/evals/src/experiment/cases.ts | 412 ++++++++++++++++++ packages/evals/src/experiment/cli.ts | 101 +++++ .../evals/src/experiment/experiment.test.ts | 96 ++++ packages/evals/src/experiment/fixtures.ts | 38 ++ packages/evals/src/experiment/main.ts | 3 + packages/evals/src/experiment/report.ts | 143 ++++++ packages/evals/src/experiment/run.ts | 111 +++++ packages/evals/src/experiment/types.ts | 118 +++++ 12 files changed, 1144 insertions(+) create mode 100644 packages/evals/scripts/experiment.mjs create mode 100644 packages/evals/src/experiment/baseline.ts create mode 100644 packages/evals/src/experiment/cases.ts create mode 100644 packages/evals/src/experiment/cli.ts create mode 100644 packages/evals/src/experiment/experiment.test.ts create mode 100644 packages/evals/src/experiment/fixtures.ts create mode 100644 packages/evals/src/experiment/main.ts create mode 100644 packages/evals/src/experiment/report.ts create mode 100644 packages/evals/src/experiment/run.ts create mode 100644 packages/evals/src/experiment/types.ts diff --git a/docs/evals.md b/docs/evals.md index f3a5caa..f0bad1e 100644 --- a/docs/evals.md +++ b/docs/evals.md @@ -29,6 +29,17 @@ ambiguous-target, missing-target, stable-prefix, and Input/Command prompt cases still required before release whenever the configured model can run; mocked or structural tests are not reported as live-model quality evidence. +## Decision and agent-context experiment + +`pnpm eval:experiment -- --strict` runs the offline experiment harness. It does not contact a model or change Lucid's runtime. +The first candidate (`current`) freezes v0.4.0's exact meta commands, conservative Command fast paths, and targeted document +context as the baseline. Cases are grouped by intent family into development, calibration, and holdout splits so paraphrases +cannot leak between them. + +Use `--suite routing|context|all`, `--split development|calibration|holdout|all`, or `--out `. By default the JSON and +Markdown reports are written under `.docdex/evals/`, which is local experiment state and must not be committed. Candidate +models remain outside the product runtime until they beat this baseline without a safety or final-task regression. + ## The bars Overall 90%. Each category 80%, and 95% for `non_destructive` and `mode_separation` (a mistake there loses the person's diff --git a/package.json b/package.json index 8ecc90f..aa97b3a 100644 --- a/package.json +++ b/package.json @@ -24,6 +24,7 @@ "mcp:build": "pnpm --filter @lucid/mcp bundle", "notices": "node scripts/notices.mjs", "eval": "node packages/evals/scripts/eval.mjs", + "eval:experiment": "node packages/evals/scripts/experiment.mjs", "dist": "pnpm --filter @lucid/desktop dist", "e2e:packaged": "pnpm --filter @lucid/desktop e2e:packaged", "dist:dir": "pnpm --filter @lucid/desktop dist:dir" diff --git a/packages/evals/scripts/experiment.mjs b/packages/evals/scripts/experiment.mjs new file mode 100644 index 0000000..4f8f914 --- /dev/null +++ b/packages/evals/scripts/experiment.mjs @@ -0,0 +1,25 @@ +import { mkdtempSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { spawnSync } from "node:child_process"; +import { build } from "esbuild"; + +const dir = mkdtempSync(join(tmpdir(), "lucid-experiment-")); +const outfile = join(dir, "experiment.mjs"); +await build({ + entryPoints: [fileURLToPath(new URL("../src/experiment/main.ts", import.meta.url))], + bundle: true, + platform: "node", + format: "esm", + target: "node22", + outfile, + banner: { + js: "import { createRequire } from 'node:module'; const require = createRequire(import.meta.url);", + }, + logLevel: "warning", +}); +const result = spawnSync(process.execPath, [outfile, ...process.argv.slice(2)], { + stdio: "inherit", +}); +process.exit(result.status ?? 1); diff --git a/packages/evals/src/experiment/baseline.ts b/packages/evals/src/experiment/baseline.ts new file mode 100644 index 0000000..afa0d6b --- /dev/null +++ b/packages/evals/src/experiment/baseline.ts @@ -0,0 +1,85 @@ +import { parseMeta } from "@lucid/core"; +import { deterministicCommand } from "@lucid/llm"; +import type { Decision, DecisionScope, RoutingCandidate, RoutingCase } from "./types"; + +const WHOLE_DOCUMENT = + /\b(?:whole|entire)\s+(?:document|page|note)|\b(?:summari[sz]e|rewrite|review)\s+(?:the\s+)?(?:document|page)\b/i; + +function scopeOf(testCase: RoutingCase): DecisionScope { + const text = testCase.utterance; + if (WHOLE_DOCUMENT.test(text)) return "document"; + if (/create an empty section/i.test(text)) return "document"; + if (/\b(?:this|selected)\s+(?:block|paragraph|heading|item)\b/i.test(text)) return "selection"; + if (/\b(?:b\d+|blocks?\s+\d+)\b/i.test(text)) return "block"; + if (/\bsection\b/i.test(text)) return "section"; + if (/\b(?:table|diagram|chart)s?\b/i.test(text)) return "component"; + if (testCase.state.cursorBlockId) return "selection"; + return "unknown"; +} + +const hintsOf = (utterance: string): string[] => [ + ...new Set([ + ...(utterance.toLowerCase().match(/\bb\d+(?:\.\d+)*\b/g) ?? []), + ...(utterance + .toLowerCase() + .match(/\b(?:first|second|third|fourth|last)\s+(?:table|diagram|chart)\b/g) ?? []), + ]), +]; + +/** A0: exactly the routing Lucid 0.4.0 already performs from explicit UI state. */ +export const currentBaseline: RoutingCandidate = { + id: "current", + decide(testCase): Decision { + const started = performance.now(); + const meta = parseMeta(testCase.utterance, testCase.state); + if (meta) { + return { + lane: "meta", + scope: "unknown", + targetHints: hintsOf(testCase.utterance), + action: "deterministic", + confidence: 1, + abstained: false, + latencyMs: performance.now() - started, + }; + } + + const scope = scopeOf(testCase); + if (testCase.state.mode === "guided") { + return { + lane: "guided", + scope: "component", + targetHints: hintsOf(testCase.utterance), + action: "deterministic", + confidence: 1, + abstained: false, + latencyMs: performance.now() - started, + }; + } + if (testCase.state.mode === "agent-read") { + return { + lane: "agent-read", + scope, + targetHints: hintsOf(testCase.utterance), + action: "read-more", + confidence: 1, + abstained: false, + latencyMs: performance.now() - started, + }; + } + + const direct = + testCase.state.mode === "command" + ? deterministicCommand(testCase.doc, testCase.utterance, testCase.state.cursorBlockId) + : undefined; + return { + lane: testCase.state.mode, + scope, + targetHints: hintsOf(testCase.utterance), + action: direct ? "deterministic" : "main-model", + confidence: 1, + abstained: false, + latencyMs: performance.now() - started, + }; + }, +}; diff --git a/packages/evals/src/experiment/cases.ts b/packages/evals/src/experiment/cases.ts new file mode 100644 index 0000000..32c3880 --- /dev/null +++ b/packages/evals/src/experiment/cases.ts @@ -0,0 +1,412 @@ +import { SAMPLE_DOC } from "@lucid/kit"; +import { contextFixture } from "./fixtures"; +import type { ContextCase, ExperimentSplit, RoutingCase } from "./types"; + +const baseState = { + proposal: false, + clarify: false, + running: false, +} as const; + +const route = ( + id: string, + family: string, + split: ExperimentSplit, + utterance: string, + state: RoutingCase["state"], + expected: RoutingCase["expected"], +): RoutingCase => ({ id, family, split, utterance, state, expected, doc: SAMPLE_DOC }); + +export const ROUTING_CASES: RoutingCase[] = [ + route( + "mode-command", + "mode-switch", + "development", + "command mode", + { ...baseState, mode: "input" }, + { lane: "meta", action: "deterministic", scope: "unknown" }, + ), + route( + "mode-input", + "mode-switch", + "development", + "back to notes", + { ...baseState, mode: "command" }, + { lane: "meta", action: "deterministic", scope: "unknown" }, + ), + route( + "mode-edit", + "mode-switch", + "development", + "let me edit", + { ...baseState, mode: "command" }, + { lane: "meta", action: "deterministic", scope: "unknown" }, + ), + route( + "build-diagram", + "guided-builder", + "calibration", + "start a diagram", + { ...baseState, mode: "input" }, + { lane: "meta", action: "deterministic", scope: "unknown" }, + ), + route( + "build-table", + "guided-builder", + "calibration", + "make a table", + { ...baseState, mode: "input" }, + { lane: "meta", action: "deterministic", scope: "unknown" }, + ), + route( + "edit-table", + "guided-builder", + "calibration", + "edit this table", + { ...baseState, mode: "command" }, + { lane: "meta", action: "deterministic", scope: "unknown" }, + ), + route( + "undo", + "history", + "holdout", + "undo the last change", + { ...baseState, mode: "command" }, + { lane: "meta", action: "deterministic", scope: "unknown" }, + ), + route( + "redo", + "history", + "holdout", + "redo that", + { ...baseState, mode: "command" }, + { lane: "meta", action: "deterministic", scope: "unknown" }, + ), + route( + "accept-proposal", + "proposal-response", + "calibration", + "looks good", + { ...baseState, mode: "command", proposal: true }, + { lane: "meta", action: "deterministic", scope: "unknown" }, + ), + route( + "reject-clarify", + "proposal-response", + "calibration", + "no thanks", + { ...baseState, mode: "command", clarify: true }, + { lane: "meta", action: "deterministic", scope: "unknown" }, + ), + route( + "choose-option", + "proposal-response", + "calibration", + "the second one", + { ...baseState, mode: "command", clarify: true }, + { lane: "meta", action: "deterministic", scope: "unknown" }, + ), + route( + "stop-running", + "running-job", + "holdout", + "stop that", + { ...baseState, mode: "command", running: true }, + { lane: "meta", action: "deterministic", scope: "unknown" }, + ), + route( + "stop-idle", + "running-job", + "holdout", + "stop that", + { ...baseState, mode: "command" }, + { lane: "command", action: "main-model", scope: "unknown" }, + ), + route( + "export-pdf", + "export", + "holdout", + "export as pdf", + { ...baseState, mode: "input" }, + { lane: "meta", action: "deterministic", scope: "unknown" }, + ), + route( + "export-markdown", + "export", + "holdout", + "save as markdown", + { ...baseState, mode: "input" }, + { lane: "meta", action: "deterministic", scope: "unknown" }, + ), + route( + "delete-selected", + "fast-delete", + "holdout", + "delete this block", + { ...baseState, mode: "command", cursorBlockId: "p1" }, + { lane: "command", action: "deterministic", scope: "selection" }, + ), + route( + "delete-without-selection", + "fast-delete", + "holdout", + "delete this block", + { ...baseState, mode: "command" }, + { lane: "command", action: "main-model", scope: "selection" }, + ), + route( + "move-up", + "fast-move", + "calibration", + "move this block up", + { ...baseState, mode: "command", cursorBlockId: "d1" }, + { lane: "command", action: "deterministic", scope: "selection" }, + ), + route( + "move-down", + "fast-move", + "calibration", + "move this block down", + { ...baseState, mode: "command", cursorBlockId: "d1" }, + { lane: "command", action: "deterministic", scope: "selection" }, + ), + route( + "move-first-up", + "fast-move", + "calibration", + "move this block up", + { ...baseState, mode: "command", cursorBlockId: "h1" }, + { lane: "command", action: "main-model", scope: "selection" }, + ), + route( + "empty-section", + "fast-section", + "holdout", + 'create an empty section named "Risks"', + { ...baseState, mode: "command" }, + { lane: "command", action: "deterministic", scope: "document" }, + ), + route( + "unquoted-section", + "fast-section", + "holdout", + "create an empty section named Risks", + { ...baseState, mode: "command" }, + { lane: "command", action: "main-model", scope: "document" }, + ), + route( + "near-delete", + "fast-near-miss", + "holdout", + "delete the selected block", + { ...baseState, mode: "command", cursorBlockId: "p1" }, + { lane: "command", action: "main-model", scope: "selection" }, + ), + route( + "near-move", + "fast-near-miss", + "holdout", + "move this paragraph up", + { ...baseState, mode: "command", cursorBlockId: "p1" }, + { lane: "command", action: "main-model", scope: "selection" }, + ), + route( + "input-note", + "ordinary-input", + "development", + "Redis entries expire after thirty days", + { ...baseState, mode: "input", cursorBlockId: "p1" }, + { lane: "input", action: "main-model", scope: "selection" }, + ), + route( + "input-table-words", + "ordinary-input", + "development", + "A table has columns latency and cost", + { ...baseState, mode: "input" }, + { lane: "input", action: "main-model", scope: "component" }, + ), + route( + "command-block-ref", + "ordinary-command", + "development", + "rewrite block 4 as a shorter comparison", + { ...baseState, mode: "command" }, + { lane: "command", action: "main-model", scope: "block" }, + ), + route( + "command-component", + "ordinary-command", + "development", + "add a consistency row to the table", + { ...baseState, mode: "command" }, + { lane: "command", action: "main-model", scope: "component" }, + ), + route( + "command-document", + "ordinary-command", + "development", + "summarize the whole document", + { ...baseState, mode: "command" }, + { lane: "command", action: "main-model", scope: "document" }, + ), + route( + "command-section", + "ordinary-command", + "development", + "shorten the Database options section", + { ...baseState, mode: "command" }, + { lane: "command", action: "main-model", scope: "section" }, + ), + route( + "guided-answer", + "active-guided", + "development", + "service", + { ...baseState, mode: "guided" }, + { lane: "guided", action: "deterministic", scope: "component" }, + ), + route( + "guided-long-answer", + "active-guided", + "development", + "The web server sends cache misses to PostgreSQL", + { ...baseState, mode: "guided" }, + { lane: "guided", action: "deterministic", scope: "component" }, + ), + route( + "agent-find-table", + "agent-read", + "calibration", + "find the SQL comparison table", + { ...baseState, mode: "agent-read" }, + { lane: "agent-read", action: "read-more", scope: "component" }, + ), + route( + "agent-whole-page", + "agent-read", + "calibration", + "read the entire page", + { ...baseState, mode: "agent-read" }, + { lane: "agent-read", action: "read-more", scope: "document" }, + ), +]; + +const sectionIds = (n: number): string[] => [ + `section-${String(n)}`, + ...Array.from({ length: 8 }, (_, i) => `topic-${String(n)}-${String(i + 1)}`), +]; +const otherSections = (n: number): string[] => + Array.from({ length: 8 }, (_, i) => i + 1) + .filter((section) => section !== n) + .flatMap(sectionIds); + +export const CONTEXT_CASES: ContextCase[] = [ + { + id: "first-table", + family: "ordered-component", + split: "holdout", + fixture: "two-tables", + utterance: "add a row to the first table", + requiredIds: ["t1"], + forbiddenIds: ["t2"], + expectedScope: "targeted", + }, + { + id: "second-table", + family: "ordered-component", + split: "holdout", + fixture: "two-tables", + utterance: "change the second table", + requiredIds: ["t2"], + forbiddenIds: ["t1"], + expectedScope: "targeted", + }, + { + id: "ambiguous-table", + family: "ambiguous-component", + split: "calibration", + fixture: "two-tables", + utterance: "change the table", + requiredIds: ["t1", "t2"], + forbiddenIds: [], + expectedScope: "targeted", + expectedAmbiguityKinds: ["table"], + }, + { + id: "missing-ref", + family: "missing-reference", + split: "holdout", + fixture: "sample", + utterance: "rewrite block 99", + requiredIds: [], + forbiddenIds: ["h1", "p1", "d1", "t1", "h2", "o1", "o2", "o3"], + expectedScope: "targeted", + expectedMissingRefs: ["b99"], + }, + { + id: "explicit-diagram-ref", + family: "explicit-reference", + split: "development", + fixture: "sample", + utterance: "make block 3 horizontal", + requiredIds: ["d1"], + forbiddenIds: ["t1"], + expectedScope: "targeted", + }, + { + id: "lexical-large-document", + family: "large-lexical", + split: "holdout", + fixture: "large-sections", + utterance: "change Quasar retention to one hundred days", + requiredIds: sectionIds(6), + forbiddenIds: otherSections(6), + expectedScope: "targeted", + }, + { + id: "cursor-large-document", + family: "large-cursor", + split: "calibration", + fixture: "large-sections", + utterance: "make this clearer", + cursorBlockId: "topic-3-4", + requiredIds: sectionIds(3), + forbiddenIds: otherSections(3), + expectedScope: "targeted", + }, + { + id: "whole-large-document", + family: "whole-document", + split: "development", + fixture: "large-sections", + utterance: "summarize the whole document", + requiredIds: Array.from({ length: 8 }, (_, i) => sectionIds(i + 1)).flat(), + forbiddenIds: [], + expectedScope: "document", + }, +]; + +export function validateExperimentCases(): string[] { + const problems: string[] = []; + const ids = new Set(); + const familySplits = new Map(); + for (const testCase of [...ROUTING_CASES, ...CONTEXT_CASES]) { + if (ids.has(testCase.id)) problems.push(`duplicate case id ${testCase.id}`); + ids.add(testCase.id); + const prior = familySplits.get(testCase.family); + if (prior && prior !== testCase.split) + problems.push(`family ${testCase.family} crosses ${prior} and ${testCase.split}`); + familySplits.set(testCase.family, testCase.split); + if (testCase.utterance.trim().length === 0) problems.push(`${testCase.id}: empty utterance`); + } + for (const testCase of CONTEXT_CASES) { + const docIds = new Set(contextFixture(testCase.fixture).blocks.map((block) => block.id)); + for (const id of [...testCase.requiredIds, ...testCase.forbiddenIds]) + if (!docIds.has(id)) problems.push(`${testCase.id}: unknown block id ${id}`); + for (const id of testCase.requiredIds) + if (testCase.forbiddenIds.includes(id)) + problems.push(`${testCase.id}: ${id} is both required and forbidden`); + } + return problems; +} diff --git a/packages/evals/src/experiment/cli.ts b/packages/evals/src/experiment/cli.ts new file mode 100644 index 0000000..f56a2ff --- /dev/null +++ b/packages/evals/src/experiment/cli.ts @@ -0,0 +1,101 @@ +import { execFileSync } from "node:child_process"; +import { mkdirSync, readFileSync, writeFileSync } from "node:fs"; +import { arch, cpus, platform, release, totalmem } from "node:os"; +import { dirname, join, resolve } from "node:path"; +import { currentBaseline } from "./baseline"; +import { CONTEXT_CASES, ROUTING_CASES, validateExperimentCases } from "./cases"; +import { experimentToMarkdown, passesExperimentGates } from "./report"; +import { runContextCase, runRoutingCase } from "./run"; +import { EXPERIMENT_DATASET_VERSION, type ExperimentResults, type ExperimentSplit } from "./types"; + +const arg = (args: string[], name: string): string | undefined => { + const index = args.indexOf(`--${name}`); + return index >= 0 ? args[index + 1] : undefined; +}; +const flag = (args: string[], name: string): boolean => args.includes(`--${name}`); + +const revision = (cwd: string): string => { + try { + return execFileSync("git", ["rev-parse", "--short=12", "HEAD"], { + cwd, + encoding: "utf8", + }).trim(); + } catch { + return "unknown"; + } +}; + +export async function experimentMain(args: string[], cwd = process.cwd()): Promise { + const candidateArg = arg(args, "candidate") ?? "current"; + const suite = arg(args, "suite") ?? "all"; + const splitArg = arg(args, "split") ?? "all"; + if (candidateArg !== "current") { + console.error("Only --candidate current exists in the baseline slice."); + return 2; + } + if (!["routing", "context", "all"].includes(suite)) { + console.error("--suite must be routing, context, or all."); + return 2; + } + if (!["development", "calibration", "holdout", "all"].includes(splitArg)) { + console.error("--split must be development, calibration, holdout, or all."); + return 2; + } + const problems = validateExperimentCases(); + if (problems.length > 0) { + console.error( + `The experiment cases have problems:\n${problems.map((p) => `- ${p}`).join("\n")}`, + ); + return 2; + } + + const split = splitArg as ExperimentSplit | "all"; + const inSplit = (testCase: { split: ExperimentSplit }): boolean => + split === "all" || testCase.split === split; + const routing = + suite === "context" + ? [] + : await Promise.all( + ROUTING_CASES.filter(inSplit).map((testCase) => + runRoutingCase(testCase, currentBaseline), + ), + ); + const context = suite === "routing" ? [] : CONTEXT_CASES.filter(inSplit).map(runContextCase); + const rootPackage = JSON.parse(readFileSync(join(cwd, "package.json"), "utf8")) as { + version: string; + }; + const cpuList = cpus(); + const results: ExperimentResults = { + meta: { + datasetVersion: EXPERIMENT_DATASET_VERSION, + candidate: currentBaseline.id, + createdAt: new Date().toISOString(), + sourceRevision: revision(cwd), + productVersion: rootPackage.version, + split, + hardware: { + platform: platform(), + release: release(), + arch: arch(), + cpu: cpuList[0]?.model ?? "unknown", + logicalCpus: cpuList.length, + totalMemoryBytes: totalmem(), + node: process.version, + }, + }, + routing, + context, + }; + const day = new Date().toLocaleDateString("en-CA"); + const output = resolve( + cwd, + arg(args, "out") ?? join(".docdex", "evals", `${day}-decision-context-current`), + ); + mkdirSync(dirname(join(output, "x")), { recursive: true }); + writeFileSync(join(output, "results.json"), `${JSON.stringify(results, null, 2)}\n`); + const markdown = experimentToMarkdown(results); + writeFileSync(join(output, "report.md"), markdown); + console.log(markdown); + console.log(`Report: ${output}`); + return flag(args, "strict") && !passesExperimentGates(results) ? 1 : 0; +} diff --git a/packages/evals/src/experiment/experiment.test.ts b/packages/evals/src/experiment/experiment.test.ts new file mode 100644 index 0000000..f5a479a --- /dev/null +++ b/packages/evals/src/experiment/experiment.test.ts @@ -0,0 +1,96 @@ +import { describe, expect, it } from "vitest"; +import { currentBaseline } from "./baseline"; +import { CONTEXT_CASES, ROUTING_CASES, validateExperimentCases } from "./cases"; +import { + experimentToMarkdown, + passesExperimentGates, + summariseContext, + summariseRouting, +} from "./report"; +import { runContextCase, runRoutingCase } from "./run"; +import type { ExperimentResults, RoutingCandidate } from "./types"; + +describe("the frozen experiment dataset", () => { + it("keeps ids unique, references valid, and families in one split", () => { + expect(validateExperimentCases()).toEqual([]); + }); + + it("has development, calibration, and holdout coverage", () => { + const splits = new Set([...ROUTING_CASES, ...CONTEXT_CASES].map(({ split }) => split)); + expect(splits).toEqual(new Set(["development", "calibration", "holdout"])); + }); +}); + +describe("the current routing baseline", () => { + it("matches the labelled routes without unsafe direct actions", async () => { + const trials = await Promise.all( + ROUTING_CASES.map((testCase) => runRoutingCase(testCase, currentBaseline)), + ); + const summary = summariseRouting(trials); + expect(summary.passRate).toBe(1); + expect(summary.highConfidenceWrongDirectActions).toBe(0); + }); + + it("detects a confident wrong direct action", async () => { + const unsafe: RoutingCandidate = { + id: "unsafe-test", + decide: () => ({ + lane: "command", + scope: "unknown", + targetHints: [], + action: "deterministic", + confidence: 0.99, + abstained: false, + latencyMs: 0, + }), + }; + const testCase = ROUTING_CASES.find(({ id }) => id === "near-delete"); + if (!testCase) throw new Error("near-delete fixture is missing"); + const trial = await runRoutingCase(testCase, unsafe); + expect(trial.highConfidenceWrongDirectAction).toBe(true); + expect(trial.pass).toBe(false); + }); +}); + +describe("targeted agent context", () => { + it("freezes the current planner's strengths and known over-selection", () => { + const trials = CONTEXT_CASES.map(runContextCase); + const summary = summariseContext(trials); + expect(summary.passRate).toBe(0.625); + expect(summary.meanTargetRecall).toBe(1); + expect(summary.forbiddenSelections).toBe(3); + expect(summary.medianEligibleContextReduction).toBeGreaterThan(0.8); + }); +}); + +describe("experiment reports", () => { + it("renders deterministic metrics and applies the baseline gates", async () => { + const results: ExperimentResults = { + meta: { + datasetVersion: 1, + candidate: "current", + createdAt: "2026-10-20T00:00:00.000Z", + sourceRevision: "abc123", + productVersion: "0.4.0", + split: "all", + hardware: { + platform: "test", + release: "test", + arch: "test", + cpu: "test", + logicalCpus: 1, + totalMemoryBytes: 1024 ** 3, + node: "v22", + }, + }, + routing: await Promise.all( + ROUTING_CASES.map((testCase) => runRoutingCase(testCase, currentBaseline)), + ), + context: CONTEXT_CASES.map(runContextCase), + }; + const markdown = experimentToMarkdown(results); + expect(markdown).toContain("High-confidence wrong direct actions: 0"); + expect(markdown).toContain("Median character reduction against full outline"); + expect(passesExperimentGates(results)).toBe(false); + }); +}); diff --git a/packages/evals/src/experiment/fixtures.ts b/packages/evals/src/experiment/fixtures.ts new file mode 100644 index 0000000..636fc58 --- /dev/null +++ b/packages/evals/src/experiment/fixtures.ts @@ -0,0 +1,38 @@ +import type { Block, LucidDoc } from "@lucid/core"; +import { SAMPLE_DOC } from "@lucid/kit"; +import { fixture } from "../fixtures"; +import type { ContextFixtureName } from "./types"; + +const clone = (value: T): T => JSON.parse(JSON.stringify(value)) as T; +const text = (value: string) => [{ type: "text" as const, text: value, styles: {} }]; + +function largeSections(): LucidDoc { + const blocks: Block[] = []; + for (let section = 1; section <= 8; section += 1) { + blocks.push({ + id: `section-${String(section)}`, + type: "heading", + props: { level: 2 }, + content: text(`Topic ${String(section)}`), + }); + for (let paragraph = 1; paragraph <= 8; paragraph += 1) { + blocks.push({ + id: `topic-${String(section)}-${String(paragraph)}`, + type: "paragraph", + props: {}, + content: text( + section === 6 && paragraph === 4 + ? "Quasar retention keeps audit events for ninety days." + : `Background note ${String(section)}.${String(paragraph)} about the architecture and its operating constraints.`, + ), + }); + } + } + return { blocks }; +} + +export function contextFixture(name: ContextFixtureName): LucidDoc { + if (name === "sample") return clone(SAMPLE_DOC); + if (name === "two-tables") return fixture("two-tables"); + return largeSections(); +} diff --git a/packages/evals/src/experiment/main.ts b/packages/evals/src/experiment/main.ts new file mode 100644 index 0000000..c5aaae3 --- /dev/null +++ b/packages/evals/src/experiment/main.ts @@ -0,0 +1,3 @@ +import { experimentMain } from "./cli"; + +process.exitCode = await experimentMain(process.argv.slice(2)); diff --git a/packages/evals/src/experiment/report.ts b/packages/evals/src/experiment/report.ts new file mode 100644 index 0000000..decf2c4 --- /dev/null +++ b/packages/evals/src/experiment/report.ts @@ -0,0 +1,143 @@ +import type { ContextTrial, ExperimentResults, RoutingTrial } from "./types"; + +const rate = (pass: number, total: number): number => (total === 0 ? 1 : pass / total); +const median = (values: number[]): number => { + if (values.length === 0) return 0; + const sorted = [...values].sort((a, b) => a - b); + const middle = Math.floor(sorted.length / 2); + return sorted.length % 2 === 0 + ? ((sorted[middle - 1] ?? 0) + (sorted[middle] ?? 0)) / 2 + : (sorted[middle] ?? 0); +}; +const percentile = (values: number[], p: number): number => { + if (values.length === 0) return 0; + const sorted = [...values].sort((a, b) => a - b); + return sorted[Math.min(sorted.length - 1, Math.ceil(sorted.length * p) - 1)] ?? 0; +}; + +export interface RoutingSummary { + total: number; + passed: number; + passRate: number; + laneAccuracy: number; + actionAccuracy: number; + scopeAccuracy: number; + highConfidenceWrongDirectActions: number; + p50LatencyMs: number; + p95LatencyMs: number; +} + +export interface ContextSummary { + total: number; + passed: number; + passRate: number; + meanTargetRecall: number; + medianContextReduction: number; + medianEligibleContextReduction: number; + forbiddenSelections: number; +} + +export function summariseRouting(trials: readonly RoutingTrial[]): RoutingSummary { + const passed = trials.filter(({ pass }) => pass).length; + const scopeTrials = trials.filter(({ expected }) => expected.scope !== undefined); + return { + total: trials.length, + passed, + passRate: rate(passed, trials.length), + laneAccuracy: rate( + trials.filter(({ decision, expected }) => decision.lane === expected.lane).length, + trials.length, + ), + actionAccuracy: rate( + trials.filter(({ decision, expected }) => decision.action === expected.action).length, + trials.length, + ), + scopeAccuracy: rate( + scopeTrials.filter(({ decision, expected }) => decision.scope === expected.scope).length, + scopeTrials.length, + ), + highConfidenceWrongDirectActions: trials.filter( + ({ highConfidenceWrongDirectAction }) => highConfidenceWrongDirectAction, + ).length, + p50LatencyMs: median(trials.map(({ decision }) => decision.latencyMs)), + p95LatencyMs: percentile( + trials.map(({ decision }) => decision.latencyMs), + 0.95, + ), + }; +} + +export function summariseContext(trials: readonly ContextTrial[]): ContextSummary { + const passed = trials.filter(({ pass }) => pass).length; + const eligible = trials.filter(({ baselineChars }) => baselineChars >= 2_000); + return { + total: trials.length, + passed, + passRate: rate(passed, trials.length), + meanTargetRecall: + trials.length === 0 + ? 1 + : trials.reduce((sum, trial) => sum + trial.targetRecall, 0) / trials.length, + medianContextReduction: median(trials.map(({ reductionRate }) => reductionRate)), + medianEligibleContextReduction: median(eligible.map(({ reductionRate }) => reductionRate)), + forbiddenSelections: trials.reduce((sum, trial) => sum + trial.forbiddenSelected.length, 0), + }; +} + +const percent = (value: number): string => `${(value * 100).toFixed(1)}%`; + +export function experimentToMarkdown(results: ExperimentResults): string { + const routing = summariseRouting(results.routing); + const context = summariseContext(results.context); + const lines = [ + `# Lucid decision/context experiment: ${results.meta.candidate}`, + "", + `- Dataset: v${String(results.meta.datasetVersion)} (${results.meta.split})`, + `- Source: ${results.meta.sourceRevision} ยท Lucid ${results.meta.productVersion}`, + `- System: ${results.meta.hardware.platform} ${results.meta.hardware.release} ${results.meta.hardware.arch}; ${results.meta.hardware.cpu}; ${String(results.meta.hardware.logicalCpus)} logical CPUs; ${(results.meta.hardware.totalMemoryBytes / 1024 ** 3).toFixed(1)} GiB`, + `- Created: ${results.meta.createdAt}`, + "", + "## Routing", + "", + `- Pass: ${String(routing.passed)}/${String(routing.total)} (${percent(routing.passRate)})`, + `- Lane/action/scope accuracy: ${percent(routing.laneAccuracy)} / ${percent(routing.actionAccuracy)} / ${percent(routing.scopeAccuracy)}`, + `- High-confidence wrong direct actions: ${String(routing.highConfidenceWrongDirectActions)}`, + `- Decision latency p50/p95: ${routing.p50LatencyMs.toFixed(3)} / ${routing.p95LatencyMs.toFixed(3)} ms`, + "", + "## Context selection", + "", + `- Pass: ${String(context.passed)}/${String(context.total)} (${percent(context.passRate)})`, + `- Mean target recall: ${percent(context.meanTargetRecall)}`, + `- Median character reduction against full outline: ${percent(context.medianContextReduction)}`, + `- Median reduction on eligible large-context tasks: ${percent(context.medianEligibleContextReduction)}`, + `- Forbidden block selections: ${String(context.forbiddenSelections)}`, + "", + "| Case | Split | Recall | Reduction | Selected | Result |", + "|---|---|---:|---:|---:|---|", + ...results.context.map( + (trial) => + `| ${trial.caseId} | ${trial.split} | ${percent(trial.targetRecall)} | ${percent(trial.reductionRate)} | ${String(trial.selectedIds.length)} | ${trial.pass ? "pass" : "FAIL"} |`, + ), + ]; + const failures = [ + ...results.routing + .filter(({ pass }) => !pass) + .map((trial) => `${trial.caseId}: ${trial.failures.join("; ")}`), + ...results.context + .filter(({ pass }) => !pass) + .map((trial) => `${trial.caseId}: ${trial.failures.join("; ")}`), + ]; + if (failures.length > 0) lines.push("", "## Failures", "", ...failures.map((f) => `- ${f}`)); + return `${lines.join("\n")}\n`; +} + +export function passesExperimentGates(results: ExperimentResults): boolean { + const routing = summariseRouting(results.routing); + const context = summariseContext(results.context); + return ( + routing.passRate === 1 && + routing.highConfidenceWrongDirectActions === 0 && + context.passRate === 1 && + context.medianContextReduction >= 0.2 + ); +} diff --git a/packages/evals/src/experiment/run.ts b/packages/evals/src/experiment/run.ts new file mode 100644 index 0000000..9e69201 --- /dev/null +++ b/packages/evals/src/experiment/run.ts @@ -0,0 +1,111 @@ +import { buildDocumentContext, buildOutline } from "@lucid/kit"; +import { contextFixture } from "./fixtures"; +import type { + ContextCase, + ContextTrial, + RoutingCandidate, + RoutingCase, + RoutingTrial, +} from "./types"; + +const sameStrings = (a: readonly string[], b: readonly string[]): boolean => { + const left = [...a].sort(); + const right = [...b].sort(); + return left.length === right.length && left.every((value, index) => value === right[index]); +}; + +export async function runRoutingCase( + testCase: RoutingCase, + candidate: RoutingCandidate, +): Promise { + const decision = await candidate.decide(testCase); + const failures: string[] = []; + if (decision.lane !== testCase.expected.lane) + failures.push(`expected lane ${testCase.expected.lane}, got ${decision.lane}`); + if (decision.action !== testCase.expected.action) + failures.push(`expected action ${testCase.expected.action}, got ${decision.action}`); + if (testCase.expected.scope && decision.scope !== testCase.expected.scope) + failures.push(`expected scope ${testCase.expected.scope}, got ${decision.scope}`); + if ( + testCase.expected.abstained !== undefined && + decision.abstained !== testCase.expected.abstained + ) + failures.push( + `expected abstained ${String(testCase.expected.abstained)}, got ${String(decision.abstained)}`, + ); + if (decision.confidence < 0 || decision.confidence > 1) + failures.push(`confidence ${String(decision.confidence)} is outside 0..1`); + if (!Number.isFinite(decision.latencyMs) || decision.latencyMs < 0) + failures.push(`latency ${String(decision.latencyMs)} is invalid`); + const highConfidenceWrongDirectAction = + decision.confidence >= 0.9 && + decision.action === "deterministic" && + testCase.expected.action !== "deterministic"; + return { + caseId: testCase.id, + family: testCase.family, + split: testCase.split, + utterance: testCase.utterance, + expected: testCase.expected, + decision, + pass: failures.length === 0, + failures, + highConfidenceWrongDirectAction, + }; +} + +export function runContextCase(testCase: ContextCase): ContextTrial { + const doc = contextFixture(testCase.fixture); + const allIds = doc.blocks.map((block) => block.id); + const baseline = buildOutline(doc, { full: allIds, maxChars: 80_000 }); + const targeted = buildDocumentContext({ + doc, + utterance: testCase.utterance, + cursorBlockId: testCase.cursorBlockId, + recentTouched: testCase.recentTouched, + }); + const selected = targeted.selection.selectedBlockIds; + const requiredHits = testCase.requiredIds.filter((id) => selected.includes(id)).length; + const targetRecall = + testCase.requiredIds.length === 0 ? 1 : requiredHits / testCase.requiredIds.length; + const targetPrecision = selected.length === 0 ? 1 : requiredHits / selected.length; + const forbiddenSelected = testCase.forbiddenIds.filter((id) => selected.includes(id)); + const missingRefs = targeted.selection.missingRefs; + const ambiguityKinds = targeted.selection.ambiguities.map(({ kind }) => kind); + const expectedMissing = testCase.expectedMissingRefs ?? []; + const expectedAmbiguity = testCase.expectedAmbiguityKinds ?? []; + const failures: string[] = []; + if (targetRecall !== 1) failures.push(`target recall was ${targetRecall.toFixed(3)}`); + if (forbiddenSelected.length > 0) + failures.push(`selected forbidden blocks: ${forbiddenSelected.join(", ")}`); + if (targeted.selection.scope !== testCase.expectedScope) + failures.push(`expected scope ${testCase.expectedScope}, got ${targeted.selection.scope}`); + if (!sameStrings(missingRefs, expectedMissing)) + failures.push( + `expected missing refs ${expectedMissing.join(", ") || "none"}, got ${missingRefs.join(", ") || "none"}`, + ); + if (!sameStrings(ambiguityKinds, expectedAmbiguity)) + failures.push( + `expected ambiguities ${expectedAmbiguity.join(", ") || "none"}, got ${ambiguityKinds.join(", ") || "none"}`, + ); + return { + caseId: testCase.id, + family: testCase.family, + split: testCase.split, + utterance: testCase.utterance, + baselineChars: baseline.text.length, + targetedChars: targeted.text.length, + reductionRate: baseline.text.length === 0 ? 0 : 1 - targeted.text.length / baseline.text.length, + selectedIds: selected, + requiredIds: testCase.requiredIds, + forbiddenIds: testCase.forbiddenIds, + targetRecall, + targetPrecision, + forbiddenSelected, + scope: targeted.selection.scope, + missingRefs, + ambiguityKinds, + pass: failures.length === 0, + failures, + }; +} diff --git a/packages/evals/src/experiment/types.ts b/packages/evals/src/experiment/types.ts new file mode 100644 index 0000000..8fab456 --- /dev/null +++ b/packages/evals/src/experiment/types.ts @@ -0,0 +1,118 @@ +import type { LucidDoc, MetaState } from "@lucid/core"; + +export const EXPERIMENT_DATASET_VERSION = 1; + +export type ExperimentSplit = "development" | "calibration" | "holdout"; +export type DecisionLane = "meta" | "guided" | "input" | "command" | "agent-read" | "clarify"; +export type DecisionScope = + "selection" | "block" | "section" | "component" | "document" | "unknown"; +export type DecisionAction = "deterministic" | "main-model" | "clarify" | "read-more"; + +export interface Decision { + lane: DecisionLane; + scope: DecisionScope; + targetHints: string[]; + action: DecisionAction; + confidence: number; + abstained: boolean; + latencyMs: number; +} + +export interface RoutingState extends MetaState { + mode: "input" | "command" | "guided" | "agent-read"; + cursorBlockId?: string; +} + +export interface RoutingCase { + id: string; + family: string; + split: ExperimentSplit; + utterance: string; + doc: LucidDoc; + state: RoutingState; + expected: { + lane: DecisionLane; + action: DecisionAction; + scope?: DecisionScope; + abstained?: boolean; + }; +} + +export interface RoutingCandidate { + id: string; + decide(testCase: RoutingCase): Promise | Decision; +} + +export type ContextFixtureName = "sample" | "two-tables" | "large-sections"; + +export interface ContextCase { + id: string; + family: string; + split: ExperimentSplit; + fixture: ContextFixtureName; + utterance: string; + cursorBlockId?: string; + recentTouched?: string[]; + requiredIds: string[]; + forbiddenIds: string[]; + expectedScope: "targeted" | "document"; + expectedMissingRefs?: string[]; + expectedAmbiguityKinds?: Array<"table" | "chart" | "diagram">; +} + +export interface RoutingTrial { + caseId: string; + family: string; + split: ExperimentSplit; + utterance: string; + expected: RoutingCase["expected"]; + decision: Decision; + pass: boolean; + failures: string[]; + highConfidenceWrongDirectAction: boolean; +} + +export interface ContextTrial { + caseId: string; + family: string; + split: ExperimentSplit; + utterance: string; + baselineChars: number; + targetedChars: number; + reductionRate: number; + selectedIds: string[]; + requiredIds: string[]; + forbiddenIds: string[]; + targetRecall: number; + targetPrecision: number; + forbiddenSelected: string[]; + scope: "targeted" | "document"; + missingRefs: string[]; + ambiguityKinds: Array<"table" | "chart" | "diagram">; + pass: boolean; + failures: string[]; +} + +export interface ExperimentMeta { + datasetVersion: number; + candidate: string; + createdAt: string; + sourceRevision: string; + productVersion: string; + split: ExperimentSplit | "all"; + hardware: { + platform: string; + release: string; + arch: string; + cpu: string; + logicalCpus: number; + totalMemoryBytes: number; + node: string; + }; +} + +export interface ExperimentResults { + meta: ExperimentMeta; + routing: RoutingTrial[]; + context: ContextTrial[]; +}