Evaluate small local decision model

docs/evals.md

@@ -63,6 +63,17 @@ hybrid never replaces Lucid's explicit-state or exact deterministic routes. Nomi
 candidate, but 55.6% semantic scope accuracy is not sufficient for product integration. Keep it experiment-only and improve
 the classifier or training examples before accepting the runtime/model-management cost.
 
+The decision-model arm uses the same safety boundary and frozen cases. With an already-installed local model, run
+`pnpm eval:experiment -- --suite routing --candidate ollama-decision --decision-model gemma3:1b`. The default confidence gate
+is 0.8; calibrate it with `--decision-min-confidence` on the calibration split before reading holdout results. This path is also
+experiment-only and never downloads or configures a model.
+
+`gemma3:1b` (815 MB) was eliminated on calibration, so its holdout was not opened. After one calibration-only prompt correction,
+it passed 12/16 routes, reached 50.0% semantic scope accuracy, and scored 0.0% on vague near-miss safety. The wrong scopes were
+reported at 95% confidence, so a confidence threshold could not preserve safety without abstaining on every useful answer.
+Semantic challenge latency was 381 ms p50 / 483 ms p95 after warm-up, and the first cold run reached 1.69 s. This is slower,
+larger, and less safely calibrated than Nomic; keep deterministic routing and reject this decision-model candidate.
+
 ## The bars
 
 Overall 90%. Each category 80%, and 95% for `non_destructive` and `mode_separation` (a mistake there loses the person's

packages/evals/src/experiment/cli.ts

@@ -5,6 +5,7 @@ import { dirname, join, resolve } from "node:path";
 import { currentBaseline } from "./baseline";
 import { CONTEXT_CASES, ROUTING_CASES, validateExperimentCases } from "./cases";
 import { currentContextStrategy, focusedContextStrategy } from "./contextStrategies";
+import { createDecisionModelRoutingCandidate } from "./decisionModelRouter";
 import { createEmbeddingRoutingCandidate } from "./embeddingRouter";
 import { experimentToMarkdown, passesExperimentGates } from "./report";
 import { runContextCase, runRoutingCases } from "./run";
@@ -34,10 +35,17 @@ export async function experimentMain(args: string[], cwd = process.cwd()): Promi
   const contextArg = arg(args, "context") ?? "current";
   const minSimilarityArg = arg(args, "embedding-min-similarity");
   const minMarginArg = arg(args, "embedding-min-margin");
+  const decisionConfidenceArg = arg(args, "decision-min-confidence");
   const minSimilarity = minSimilarityArg === undefined ? undefined : Number(minSimilarityArg);
   const minMargin = minMarginArg === undefined ? undefined : Number(minMarginArg);
-  if (candidateArg !== "current" && candidateArg !== "ollama-embedding") {
-    console.error("--candidate must be current or ollama-embedding.");
+  const decisionConfidence =
+    decisionConfidenceArg === undefined ? undefined : Number(decisionConfidenceArg);
+  if (
+    candidateArg !== "current" &&
+    candidateArg !== "ollama-embedding" &&
+    candidateArg !== "ollama-decision"
+  ) {
+    console.error("--candidate must be current, ollama-embedding, or ollama-decision.");
     return 2;
   }
   if (!["routing", "context", "all"].includes(suite)) {
@@ -60,6 +68,13 @@ export async function experimentMain(args: string[], cwd = process.cwd()): Promi
     console.error("--embedding-min-margin must be a non-negative number.");
     return 2;
   }
+  if (
+    decisionConfidence !== undefined &&
+    (!Number.isFinite(decisionConfidence) || decisionConfidence < 0 || decisionConfidence > 1)
+  ) {
+    console.error("--decision-min-confidence must be between 0 and 1.");
+    return 2;
+  }
   const problems = validateExperimentCases();
   if (problems.length > 0) {
     console.error(
@@ -80,7 +95,13 @@ export async function experimentMain(args: string[], cwd = process.cwd()): Promi
           ...(minSimilarity === undefined ? {} : { minSimilarity }),
           ...(minMargin === undefined ? {} : { minMargin }),
         })
-      : currentBaseline;
+      : candidateArg === "ollama-decision"
+        ? createDecisionModelRoutingCandidate({
+            model: arg(args, "decision-model") ?? "gemma3:1b",
+            ...(ollamaUrl ? { baseUrl: ollamaUrl } : {}),
+            ...(decisionConfidence === undefined ? {} : { minConfidence: decisionConfidence }),
+          })
+        : currentBaseline;
   const routing =
     suite === "context"
       ? []

packages/evals/src/experiment/decisionModelRouter.ts

@@ -0,0 +1,116 @@
+import { currentBaseline } from "./baseline";
+import type { DecisionScope, RoutingCandidate } from "./types";
+
+const SCOPES: DecisionScope[] = [
+  "document",
+  "section",
+  "component",
+  "block",
+  "selection",
+  "unknown",
+];
+
+export interface ScopeDecision {
+  scope: DecisionScope;
+  confidence: number;
+}
+
+export type DecideScope = (utterance: string) => Promise<ScopeDecision>;
+
+const SYSTEM_PROMPT = `Classify only the target scope of a note-editing request.
+document = all content or the note as a whole
+section = one topic group, chapter, or named portion
+component = one table, diagram, chart, grid, or visual
+block = one paragraph, heading, numbered item, or content unit
+selection = explicitly highlighted or selected text
+unknown = no clear target; vague pronouns without an actual UI selection are unknown
+Prefer unknown whenever the target is unclear. Never infer document merely because no smaller target is named.
+Examples:
+"make this better" -> unknown
+"continue writing" -> unknown
+"what is noteworthy?" -> unknown
+"shorten the portion about pricing" -> section
+"update the comparison matrix" -> component
+"rewrite paragraph twelve" -> block
+Return only the requested JSON. Do not perform the edit.`;
+
+export function createOllamaScopeDecider(
+  model: string,
+  baseUrl = "http://127.0.0.1:11434",
+): DecideScope {
+  return async (utterance) => {
+    const response = await fetch(`${baseUrl.replace(/\/$/, "")}/api/chat`, {
+      method: "POST",
+      headers: { "content-type": "application/json" },
+      body: JSON.stringify({
+        model,
+        stream: false,
+        messages: [
+          { role: "system", content: SYSTEM_PROMPT },
+          { role: "user", content: utterance },
+        ],
+        format: {
+          type: "object",
+          properties: {
+            scope: { type: "string", enum: SCOPES },
+            confidence: { type: "number", minimum: 0, maximum: 1 },
+          },
+          required: ["scope", "confidence"],
+          additionalProperties: false,
+        },
+        options: { temperature: 0, num_predict: 32 },
+      }),
+    });
+    if (!response.ok) {
+      throw new Error(`Ollama decision request failed (${String(response.status)}).`);
+    }
+    const body = (await response.json()) as { message?: { content?: string } };
+    const content = body.message?.content;
+    if (!content) throw new Error("Ollama returned an empty decision response.");
+    const parsed = JSON.parse(content) as { scope?: unknown; confidence?: unknown };
+    if (typeof parsed.scope !== "string" || !SCOPES.includes(parsed.scope as DecisionScope)) {
+      throw new Error("Ollama returned an invalid decision scope.");
+    }
+    if (
+      typeof parsed.confidence !== "number" ||
+      !Number.isFinite(parsed.confidence) ||
+      parsed.confidence < 0 ||
+      parsed.confidence > 1
+    ) {
+      throw new Error("Ollama returned an invalid decision confidence.");
+    }
+    return { scope: parsed.scope as DecisionScope, confidence: parsed.confidence };
+  };
+}
+
+export function createDecisionModelRoutingCandidate(options: {
+  model: string;
+  decideScope?: DecideScope;
+  baseUrl?: string;
+  minConfidence?: number;
+}): RoutingCandidate {
+  const decideScope =
+    options.decideScope ?? createOllamaScopeDecider(options.model, options.baseUrl);
+  const minConfidence = options.minConfidence ?? 0.8;
+  return {
+    id: `ollama-decision:${options.model}`,
+    async decide(testCase) {
+      const started = performance.now();
+      const baseline = await currentBaseline.decide(testCase);
+      if (baseline.action !== "main-model" || baseline.scope !== "unknown") {
+        return { ...baseline, latencyMs: performance.now() - started };
+      }
+      const raw = await decideScope(testCase.utterance);
+      const scope =
+        raw.scope === "unknown" || raw.confidence < minConfidence ? "unknown" : raw.scope;
+      return {
+        ...baseline,
+        scope,
+        confidence: raw.confidence,
+        abstained: scope === "unknown",
+        latencyMs: performance.now() - started,
+        decisionModel: { rawScope: raw.scope, rawConfidence: raw.confidence },
+      };
+    },
+  };
+}

packages/evals/src/experiment/experiment.test.ts

@@ -2,6 +2,7 @@ import { describe, expect, it } from "vitest";
 import { currentBaseline } from "./baseline";
 import { CONTEXT_CASES, ROUTING_CASES, validateExperimentCases } from "./cases";
 import { currentContextStrategy, focusedContextStrategy } from "./contextStrategies";
+import { createDecisionModelRoutingCandidate } from "./decisionModelRouter";
 import { classifyScope, createEmbeddingRoutingCandidate } from "./embeddingRouter";
 import {
   experimentToMarkdown,
@@ -93,6 +94,28 @@ describe("the optional embedding routing candidate", () => {
   });
 });
 
+describe("the optional local decision-model candidate", () => {
+  it("never invokes the model for an existing deterministic route", async () => {
+    const candidate = createDecisionModelRoutingCandidate({
+      model: "test",
+      decideScope: () => Promise.reject(new Error("decision model should not run")),
+    });
+    const testCase = ROUTING_CASES.find(({ id }) => id === "mode-command");
+    if (!testCase) throw new Error("mode-command fixture is missing");
+    expect((await runRoutingCase(testCase, candidate)).pass).toBe(true);
+  });
+
+  it("accepts a confident scope classification for an unknown baseline scope", async () => {
+    const candidate = createDecisionModelRoutingCandidate({
+      model: "test",
+      decideScope: () => Promise.resolve({ scope: "section", confidence: 0.9 }),
+    });
+    const testCase = ROUTING_CASES.find(({ id }) => id === "semantic-section-database");
+    if (!testCase) throw new Error("semantic-section-database fixture is missing");
+    expect((await runRoutingCase(testCase, candidate)).pass).toBe(true);
+  });
+});
+
 describe("targeted agent context", () => {
   it("freezes the current planner's strengths and known over-selection", () => {
     const trials = CONTEXT_CASES.map((testCase) =>

packages/evals/src/experiment/report.ts

@@ -107,6 +107,12 @@ export function summariseContext(trials: readonly ContextTrial[]): ContextSummar
 }
 
 const percent = (value: number): string => `${(value * 100).toFixed(1)}%`;
+const routingEvidence = (trial: RoutingTrial): string => {
+  if (trial.decision.embedding) return `margin ${trial.decision.embedding.margin.toFixed(4)}`;
+  if (trial.decision.decisionModel)
+    return `${trial.decision.decisionModel.rawScope} @ ${percent(trial.decision.decisionModel.rawConfidence)}`;
+  return "-";
+};
 
 export function experimentToMarkdown(results: ExperimentResults): string {
   const routing = summariseRouting(results.routing);
@@ -129,13 +135,13 @@ export function experimentToMarkdown(results: ExperimentResults): string {
     `- Decision latency p50/p95/max: ${routing.p50LatencyMs.toFixed(3)} / ${routing.p95LatencyMs.toFixed(3)} / ${routing.maxLatencyMs.toFixed(3)} ms`,
     `- Semantic challenge latency p50/p95: ${routing.challengeP50LatencyMs.toFixed(3)} / ${routing.challengeP95LatencyMs.toFixed(3)} ms`,
     "",
-    "| Challenge | Expected | Predicted | Confidence | Margin | Result |",
+    "| Challenge | Expected | Predicted | Confidence | Evidence | Result |",
     "|---|---|---|---:|---:|---|",
     ...results.routing
       .filter(({ challenge }) => challenge !== undefined)
       .map(
         (trial) =>
-          `| ${trial.caseId} | ${trial.expected.scope ?? "-"} | ${trial.decision.scope} | ${percent(trial.decision.confidence)} | ${trial.decision.embedding?.margin.toFixed(4) ?? "-"} | ${trial.pass ? "pass" : "FAIL"} |`,
+          `| ${trial.caseId} | ${trial.expected.scope ?? "-"} | ${trial.decision.scope} | ${percent(trial.decision.confidence)} | ${routingEvidence(trial)} | ${trial.pass ? "pass" : "FAIL"} |`,
       ),
     "",
     "## Context selection",

packages/evals/src/experiment/types.ts

@@ -17,6 +17,7 @@ export interface Decision {
   abstained: boolean;
   latencyMs: number;
   embedding?: { similarity: number; margin: number };
+  decisionModel?: { rawScope: DecisionScope; rawConfidence: number };
 }
 
 export interface RoutingState extends MetaState {