From f5cb9e3ec23278b90df40a02bd3da676845c93e4 Mon Sep 17 00:00:00 2001 From: Omkar Tripathi <38633467+triggeredcode@users.noreply.github.com> Date: Tue, 20 Oct 2026 16:14:29 +0530 Subject: [PATCH] feat(evals): evaluate local decision model (#57) --- docs/evals.md | 11 ++ packages/evals/src/experiment/cli.ts | 27 +++- .../src/experiment/decisionModelRouter.ts | 116 ++++++++++++++++++ .../evals/src/experiment/experiment.test.ts | 23 ++++ packages/evals/src/experiment/report.ts | 10 +- packages/evals/src/experiment/types.ts | 1 + 6 files changed, 183 insertions(+), 5 deletions(-) create mode 100644 packages/evals/src/experiment/decisionModelRouter.ts diff --git a/docs/evals.md b/docs/evals.md index d3c4502..875d5ba 100644 --- a/docs/evals.md +++ b/docs/evals.md @@ -63,6 +63,17 @@ hybrid never replaces Lucid's explicit-state or exact deterministic routes. Nomi candidate, but 55.6% semantic scope accuracy is not sufficient for product integration. Keep it experiment-only and improve the classifier or training examples before accepting the runtime/model-management cost. +The decision-model arm uses the same safety boundary and frozen cases. With an already-installed local model, run +`pnpm eval:experiment -- --suite routing --candidate ollama-decision --decision-model gemma3:1b`. The default confidence gate +is 0.8; calibrate it with `--decision-min-confidence` on the calibration split before reading holdout results. This path is also +experiment-only and never downloads or configures a model. + +`gemma3:1b` (815 MB) was eliminated on calibration, so its holdout was not opened. After one calibration-only prompt correction, +it passed 12/16 routes, reached 50.0% semantic scope accuracy, and scored 0.0% on vague near-miss safety. The wrong scopes were +reported at 95% confidence, so a confidence threshold could not preserve safety without abstaining on every useful answer. +Semantic challenge latency was 381 ms p50 / 483 ms p95 after warm-up, and the first cold run reached 1.69 s. This is slower, +larger, and less safely calibrated than Nomic; keep deterministic routing and reject this decision-model candidate. + ## The bars Overall 90%. Each category 80%, and 95% for `non_destructive` and `mode_separation` (a mistake there loses the person's diff --git a/packages/evals/src/experiment/cli.ts b/packages/evals/src/experiment/cli.ts index 63109bf..2bf0512 100644 --- a/packages/evals/src/experiment/cli.ts +++ b/packages/evals/src/experiment/cli.ts @@ -5,6 +5,7 @@ import { dirname, join, resolve } from "node:path"; import { currentBaseline } from "./baseline"; import { CONTEXT_CASES, ROUTING_CASES, validateExperimentCases } from "./cases"; import { currentContextStrategy, focusedContextStrategy } from "./contextStrategies"; +import { createDecisionModelRoutingCandidate } from "./decisionModelRouter"; import { createEmbeddingRoutingCandidate } from "./embeddingRouter"; import { experimentToMarkdown, passesExperimentGates } from "./report"; import { runContextCase, runRoutingCases } from "./run"; @@ -34,10 +35,17 @@ export async function experimentMain(args: string[], cwd = process.cwd()): Promi const contextArg = arg(args, "context") ?? "current"; const minSimilarityArg = arg(args, "embedding-min-similarity"); const minMarginArg = arg(args, "embedding-min-margin"); + const decisionConfidenceArg = arg(args, "decision-min-confidence"); const minSimilarity = minSimilarityArg === undefined ? undefined : Number(minSimilarityArg); const minMargin = minMarginArg === undefined ? undefined : Number(minMarginArg); - if (candidateArg !== "current" && candidateArg !== "ollama-embedding") { - console.error("--candidate must be current or ollama-embedding."); + const decisionConfidence = + decisionConfidenceArg === undefined ? undefined : Number(decisionConfidenceArg); + if ( + candidateArg !== "current" && + candidateArg !== "ollama-embedding" && + candidateArg !== "ollama-decision" + ) { + console.error("--candidate must be current, ollama-embedding, or ollama-decision."); return 2; } if (!["routing", "context", "all"].includes(suite)) { @@ -60,6 +68,13 @@ export async function experimentMain(args: string[], cwd = process.cwd()): Promi console.error("--embedding-min-margin must be a non-negative number."); return 2; } + if ( + decisionConfidence !== undefined && + (!Number.isFinite(decisionConfidence) || decisionConfidence < 0 || decisionConfidence > 1) + ) { + console.error("--decision-min-confidence must be between 0 and 1."); + return 2; + } const problems = validateExperimentCases(); if (problems.length > 0) { console.error( @@ -80,7 +95,13 @@ export async function experimentMain(args: string[], cwd = process.cwd()): Promi ...(minSimilarity === undefined ? {} : { minSimilarity }), ...(minMargin === undefined ? {} : { minMargin }), }) - : currentBaseline; + : candidateArg === "ollama-decision" + ? createDecisionModelRoutingCandidate({ + model: arg(args, "decision-model") ?? "gemma3:1b", + ...(ollamaUrl ? { baseUrl: ollamaUrl } : {}), + ...(decisionConfidence === undefined ? {} : { minConfidence: decisionConfidence }), + }) + : currentBaseline; const routing = suite === "context" ? [] diff --git a/packages/evals/src/experiment/decisionModelRouter.ts b/packages/evals/src/experiment/decisionModelRouter.ts new file mode 100644 index 0000000..92b78d6 --- /dev/null +++ b/packages/evals/src/experiment/decisionModelRouter.ts @@ -0,0 +1,116 @@ +import { currentBaseline } from "./baseline"; +import type { DecisionScope, RoutingCandidate } from "./types"; + +const SCOPES: DecisionScope[] = [ + "document", + "section", + "component", + "block", + "selection", + "unknown", +]; + +export interface ScopeDecision { + scope: DecisionScope; + confidence: number; +} + +export type DecideScope = (utterance: string) => Promise; + +const SYSTEM_PROMPT = `Classify only the target scope of a note-editing request. +document = all content or the note as a whole +section = one topic group, chapter, or named portion +component = one table, diagram, chart, grid, or visual +block = one paragraph, heading, numbered item, or content unit +selection = explicitly highlighted or selected text +unknown = no clear target; vague pronouns without an actual UI selection are unknown +Prefer unknown whenever the target is unclear. Never infer document merely because no smaller target is named. +Examples: +"make this better" -> unknown +"continue writing" -> unknown +"what is noteworthy?" -> unknown +"shorten the portion about pricing" -> section +"update the comparison matrix" -> component +"rewrite paragraph twelve" -> block +Return only the requested JSON. Do not perform the edit.`; + +export function createOllamaScopeDecider( + model: string, + baseUrl = "http://127.0.0.1:11434", +): DecideScope { + return async (utterance) => { + const response = await fetch(`${baseUrl.replace(/\/$/, "")}/api/chat`, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ + model, + stream: false, + messages: [ + { role: "system", content: SYSTEM_PROMPT }, + { role: "user", content: utterance }, + ], + format: { + type: "object", + properties: { + scope: { type: "string", enum: SCOPES }, + confidence: { type: "number", minimum: 0, maximum: 1 }, + }, + required: ["scope", "confidence"], + additionalProperties: false, + }, + options: { temperature: 0, num_predict: 32 }, + }), + }); + if (!response.ok) { + throw new Error(`Ollama decision request failed (${String(response.status)}).`); + } + const body = (await response.json()) as { message?: { content?: string } }; + const content = body.message?.content; + if (!content) throw new Error("Ollama returned an empty decision response."); + const parsed = JSON.parse(content) as { scope?: unknown; confidence?: unknown }; + if (typeof parsed.scope !== "string" || !SCOPES.includes(parsed.scope as DecisionScope)) { + throw new Error("Ollama returned an invalid decision scope."); + } + if ( + typeof parsed.confidence !== "number" || + !Number.isFinite(parsed.confidence) || + parsed.confidence < 0 || + parsed.confidence > 1 + ) { + throw new Error("Ollama returned an invalid decision confidence."); + } + return { scope: parsed.scope as DecisionScope, confidence: parsed.confidence }; + }; +} + +export function createDecisionModelRoutingCandidate(options: { + model: string; + decideScope?: DecideScope; + baseUrl?: string; + minConfidence?: number; +}): RoutingCandidate { + const decideScope = + options.decideScope ?? createOllamaScopeDecider(options.model, options.baseUrl); + const minConfidence = options.minConfidence ?? 0.8; + return { + id: `ollama-decision:${options.model}`, + async decide(testCase) { + const started = performance.now(); + const baseline = await currentBaseline.decide(testCase); + if (baseline.action !== "main-model" || baseline.scope !== "unknown") { + return { ...baseline, latencyMs: performance.now() - started }; + } + const raw = await decideScope(testCase.utterance); + const scope = + raw.scope === "unknown" || raw.confidence < minConfidence ? "unknown" : raw.scope; + return { + ...baseline, + scope, + confidence: raw.confidence, + abstained: scope === "unknown", + latencyMs: performance.now() - started, + decisionModel: { rawScope: raw.scope, rawConfidence: raw.confidence }, + }; + }, + }; +} diff --git a/packages/evals/src/experiment/experiment.test.ts b/packages/evals/src/experiment/experiment.test.ts index 0da640a..08b8c26 100644 --- a/packages/evals/src/experiment/experiment.test.ts +++ b/packages/evals/src/experiment/experiment.test.ts @@ -2,6 +2,7 @@ import { describe, expect, it } from "vitest"; import { currentBaseline } from "./baseline"; import { CONTEXT_CASES, ROUTING_CASES, validateExperimentCases } from "./cases"; import { currentContextStrategy, focusedContextStrategy } from "./contextStrategies"; +import { createDecisionModelRoutingCandidate } from "./decisionModelRouter"; import { classifyScope, createEmbeddingRoutingCandidate } from "./embeddingRouter"; import { experimentToMarkdown, @@ -93,6 +94,28 @@ describe("the optional embedding routing candidate", () => { }); }); +describe("the optional local decision-model candidate", () => { + it("never invokes the model for an existing deterministic route", async () => { + const candidate = createDecisionModelRoutingCandidate({ + model: "test", + decideScope: () => Promise.reject(new Error("decision model should not run")), + }); + const testCase = ROUTING_CASES.find(({ id }) => id === "mode-command"); + if (!testCase) throw new Error("mode-command fixture is missing"); + expect((await runRoutingCase(testCase, candidate)).pass).toBe(true); + }); + + it("accepts a confident scope classification for an unknown baseline scope", async () => { + const candidate = createDecisionModelRoutingCandidate({ + model: "test", + decideScope: () => Promise.resolve({ scope: "section", confidence: 0.9 }), + }); + const testCase = ROUTING_CASES.find(({ id }) => id === "semantic-section-database"); + if (!testCase) throw new Error("semantic-section-database fixture is missing"); + expect((await runRoutingCase(testCase, candidate)).pass).toBe(true); + }); +}); + describe("targeted agent context", () => { it("freezes the current planner's strengths and known over-selection", () => { const trials = CONTEXT_CASES.map((testCase) => diff --git a/packages/evals/src/experiment/report.ts b/packages/evals/src/experiment/report.ts index 52f63c0..4948e72 100644 --- a/packages/evals/src/experiment/report.ts +++ b/packages/evals/src/experiment/report.ts @@ -107,6 +107,12 @@ export function summariseContext(trials: readonly ContextTrial[]): ContextSummar } const percent = (value: number): string => `${(value * 100).toFixed(1)}%`; +const routingEvidence = (trial: RoutingTrial): string => { + if (trial.decision.embedding) return `margin ${trial.decision.embedding.margin.toFixed(4)}`; + if (trial.decision.decisionModel) + return `${trial.decision.decisionModel.rawScope} @ ${percent(trial.decision.decisionModel.rawConfidence)}`; + return "-"; +}; export function experimentToMarkdown(results: ExperimentResults): string { const routing = summariseRouting(results.routing); @@ -129,13 +135,13 @@ export function experimentToMarkdown(results: ExperimentResults): string { `- Decision latency p50/p95/max: ${routing.p50LatencyMs.toFixed(3)} / ${routing.p95LatencyMs.toFixed(3)} / ${routing.maxLatencyMs.toFixed(3)} ms`, `- Semantic challenge latency p50/p95: ${routing.challengeP50LatencyMs.toFixed(3)} / ${routing.challengeP95LatencyMs.toFixed(3)} ms`, "", - "| Challenge | Expected | Predicted | Confidence | Margin | Result |", + "| Challenge | Expected | Predicted | Confidence | Evidence | Result |", "|---|---|---|---:|---:|---|", ...results.routing .filter(({ challenge }) => challenge !== undefined) .map( (trial) => - `| ${trial.caseId} | ${trial.expected.scope ?? "-"} | ${trial.decision.scope} | ${percent(trial.decision.confidence)} | ${trial.decision.embedding?.margin.toFixed(4) ?? "-"} | ${trial.pass ? "pass" : "FAIL"} |`, + `| ${trial.caseId} | ${trial.expected.scope ?? "-"} | ${trial.decision.scope} | ${percent(trial.decision.confidence)} | ${routingEvidence(trial)} | ${trial.pass ? "pass" : "FAIL"} |`, ), "", "## Context selection", diff --git a/packages/evals/src/experiment/types.ts b/packages/evals/src/experiment/types.ts index 2c17a88..0c876b2 100644 --- a/packages/evals/src/experiment/types.ts +++ b/packages/evals/src/experiment/types.ts @@ -17,6 +17,7 @@ export interface Decision { abstained: boolean; latencyMs: number; embedding?: { similarity: number; margin: number }; + decisionModel?: { rawScope: DecisionScope; rawConfidence: number }; } export interface RoutingState extends MetaState {