Add routing and agent-context experiment baseline

docs/evals.md

@@ -29,6 +29,17 @@ ambiguous-target, missing-target, stable-prefix, and Input/Command prompt cases
 still required before release whenever the configured model can run; mocked or structural tests are not reported as live-model
 quality evidence.
 
+## Decision and agent-context experiment
+
+`pnpm eval:experiment -- --strict` runs the offline experiment harness. It does not contact a model or change Lucid's runtime.
+The first candidate (`current`) freezes v0.4.0's exact meta commands, conservative Command fast paths, and targeted document
+context as the baseline. Cases are grouped by intent family into development, calibration, and holdout splits so paraphrases
+cannot leak between them.
+
+Use `--suite routing|context|all`, `--split development|calibration|holdout|all`, or `--out <dir>`. By default the JSON and
+Markdown reports are written under `.docdex/evals/`, which is local experiment state and must not be committed. Candidate
+models remain outside the product runtime until they beat this baseline without a safety or final-task regression.
+
 ## The bars
 
 Overall 90%. Each category 80%, and 95% for `non_destructive` and `mode_separation` (a mistake there loses the person's

package.json

@@ -24,6 +24,7 @@
     "mcp:build": "pnpm --filter @lucid/mcp bundle",
     "notices": "node scripts/notices.mjs",
     "eval": "node packages/evals/scripts/eval.mjs",
+    "eval:experiment": "node packages/evals/scripts/experiment.mjs",
     "dist": "pnpm --filter @lucid/desktop dist",
     "e2e:packaged": "pnpm --filter @lucid/desktop e2e:packaged",
     "dist:dir": "pnpm --filter @lucid/desktop dist:dir"

packages/evals/scripts/experiment.mjs

@@ -0,0 +1,25 @@
+import { mkdtempSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { fileURLToPath } from "node:url";
+import { spawnSync } from "node:child_process";
+import { build } from "esbuild";
+
+const dir = mkdtempSync(join(tmpdir(), "lucid-experiment-"));
+const outfile = join(dir, "experiment.mjs");
+await build({
+  entryPoints: [fileURLToPath(new URL("../src/experiment/main.ts", import.meta.url))],
+  bundle: true,
+  platform: "node",
+  format: "esm",
+  target: "node22",
+  outfile,
+  banner: {
+    js: "import { createRequire } from 'node:module'; const require = createRequire(import.meta.url);",
+  },
+  logLevel: "warning",
+});
+const result = spawnSync(process.execPath, [outfile, ...process.argv.slice(2)], {
+  stdio: "inherit",
+});
+process.exit(result.status ?? 1);

packages/evals/src/experiment/baseline.ts

@@ -0,0 +1,85 @@
+import { parseMeta } from "@lucid/core";
+import { deterministicCommand } from "@lucid/llm";
+import type { Decision, DecisionScope, RoutingCandidate, RoutingCase } from "./types";
+
+const WHOLE_DOCUMENT =
+  /\b(?:whole|entire)\s+(?:document|page|note)|\b(?:summari[sz]e|rewrite|review)\s+(?:the\s+)?(?:document|page)\b/i;
+
+function scopeOf(testCase: RoutingCase): DecisionScope {
+  const text = testCase.utterance;
+  if (WHOLE_DOCUMENT.test(text)) return "document";
+  if (/create an empty section/i.test(text)) return "document";
+  if (/\b(?:this|selected)\s+(?:block|paragraph|heading|item)\b/i.test(text)) return "selection";
+  if (/\b(?:b\d+|blocks?\s+\d+)\b/i.test(text)) return "block";
+  if (/\bsection\b/i.test(text)) return "section";
+  if (/\b(?:table|diagram|chart)s?\b/i.test(text)) return "component";
+  if (testCase.state.cursorBlockId) return "selection";
+  return "unknown";
+}
+
+const hintsOf = (utterance: string): string[] => [
+  ...new Set([
+    ...(utterance.toLowerCase().match(/\bb\d+(?:\.\d+)*\b/g) ?? []),
+    ...(utterance
+      .toLowerCase()
+      .match(/\b(?:first|second|third|fourth|last)\s+(?:table|diagram|chart)\b/g) ?? []),
+  ]),
+];
+
+/** A0: exactly the routing Lucid 0.4.0 already performs from explicit UI state. */
+export const currentBaseline: RoutingCandidate = {
+  id: "current",
+  decide(testCase): Decision {
+    const started = performance.now();
+    const meta = parseMeta(testCase.utterance, testCase.state);
+    if (meta) {
+      return {
+        lane: "meta",
+        scope: "unknown",
+        targetHints: hintsOf(testCase.utterance),
+        action: "deterministic",
+        confidence: 1,
+        abstained: false,
+        latencyMs: performance.now() - started,
+      };
+    }
+
+    const scope = scopeOf(testCase);
+    if (testCase.state.mode === "guided") {
+      return {
+        lane: "guided",
+        scope: "component",
+        targetHints: hintsOf(testCase.utterance),
+        action: "deterministic",
+        confidence: 1,
+        abstained: false,
+        latencyMs: performance.now() - started,
+      };
+    }
+    if (testCase.state.mode === "agent-read") {
+      return {
+        lane: "agent-read",
+        scope,
+        targetHints: hintsOf(testCase.utterance),
+        action: "read-more",
+        confidence: 1,
+        abstained: false,
+        latencyMs: performance.now() - started,
+      };
+    }
+
+    const direct =
+      testCase.state.mode === "command"
+        ? deterministicCommand(testCase.doc, testCase.utterance, testCase.state.cursorBlockId)
+        : undefined;
+    return {
+      lane: testCase.state.mode,
+      scope,
+      targetHints: hintsOf(testCase.utterance),
+      action: direct ? "deterministic" : "main-model",
+      confidence: 1,
+      abstained: false,
+      latencyMs: performance.now() - started,
+    };
+  },
+};