diff --git a/docs/evals.md b/docs/evals.md index f0bad1e..90a177e 100644 --- a/docs/evals.md +++ b/docs/evals.md @@ -36,9 +36,11 @@ The first candidate (`current`) freezes v0.4.0's exact meta commands, conservati context as the baseline. Cases are grouped by intent family into development, calibration, and holdout splits so paraphrases cannot leak between them. -Use `--suite routing|context|all`, `--split development|calibration|holdout|all`, or `--out `. By default the JSON and -Markdown reports are written under `.docdex/evals/`, which is local experiment state and must not be committed. Candidate -models remain outside the product runtime until they beat this baseline without a safety or final-task regression. +Use `--suite routing|context|all`, `--split development|calibration|holdout|all`, `--context current|focused`, or `--out `. +The `focused` context candidate is experiment-only: it keeps Lucid's target resolution but limits resolved reads to the target +and its section heading. By default JSON and Markdown reports are written under `.docdex/evals/`, which is local experiment +state and must not be committed. Candidates remain outside the product runtime until they beat the baseline without a safety +or final-task regression. ## The bars diff --git a/packages/evals/src/experiment/cases.ts b/packages/evals/src/experiment/cases.ts index 32c3880..eef958a 100644 --- a/packages/evals/src/experiment/cases.ts +++ b/packages/evals/src/experiment/cases.ts @@ -360,7 +360,7 @@ export const CONTEXT_CASES: ContextCase[] = [ split: "holdout", fixture: "large-sections", utterance: "change Quasar retention to one hundred days", - requiredIds: sectionIds(6), + requiredIds: ["section-6", "topic-6-4"], forbiddenIds: otherSections(6), expectedScope: "targeted", }, @@ -371,7 +371,7 @@ export const CONTEXT_CASES: ContextCase[] = [ fixture: "large-sections", utterance: "make this clearer", cursorBlockId: "topic-3-4", - requiredIds: sectionIds(3), + requiredIds: ["section-3", "topic-3-4"], forbiddenIds: otherSections(3), expectedScope: "targeted", }, diff --git a/packages/evals/src/experiment/cli.ts b/packages/evals/src/experiment/cli.ts index f56a2ff..65fa34f 100644 --- a/packages/evals/src/experiment/cli.ts +++ b/packages/evals/src/experiment/cli.ts @@ -4,6 +4,7 @@ import { arch, cpus, platform, release, totalmem } from "node:os"; import { dirname, join, resolve } from "node:path"; import { currentBaseline } from "./baseline"; import { CONTEXT_CASES, ROUTING_CASES, validateExperimentCases } from "./cases"; +import { currentContextStrategy, focusedContextStrategy } from "./contextStrategies"; import { experimentToMarkdown, passesExperimentGates } from "./report"; import { runContextCase, runRoutingCase } from "./run"; import { EXPERIMENT_DATASET_VERSION, type ExperimentResults, type ExperimentSplit } from "./types"; @@ -29,6 +30,7 @@ export async function experimentMain(args: string[], cwd = process.cwd()): Promi const candidateArg = arg(args, "candidate") ?? "current"; const suite = arg(args, "suite") ?? "all"; const splitArg = arg(args, "split") ?? "all"; + const contextArg = arg(args, "context") ?? "current"; if (candidateArg !== "current") { console.error("Only --candidate current exists in the baseline slice."); return 2; @@ -41,6 +43,10 @@ export async function experimentMain(args: string[], cwd = process.cwd()): Promi console.error("--split must be development, calibration, holdout, or all."); return 2; } + if (contextArg !== "current" && contextArg !== "focused") { + console.error("--context must be current or focused."); + return 2; + } const problems = validateExperimentCases(); if (problems.length > 0) { console.error( @@ -60,7 +66,12 @@ export async function experimentMain(args: string[], cwd = process.cwd()): Promi runRoutingCase(testCase, currentBaseline), ), ); - const context = suite === "routing" ? [] : CONTEXT_CASES.filter(inSplit).map(runContextCase); + const contextStrategy = + contextArg === "focused" ? focusedContextStrategy : currentContextStrategy; + const context = + suite === "routing" + ? [] + : CONTEXT_CASES.filter(inSplit).map((testCase) => runContextCase(testCase, contextStrategy)); const rootPackage = JSON.parse(readFileSync(join(cwd, "package.json"), "utf8")) as { version: string; }; @@ -69,6 +80,7 @@ export async function experimentMain(args: string[], cwd = process.cwd()): Promi meta: { datasetVersion: EXPERIMENT_DATASET_VERSION, candidate: currentBaseline.id, + contextStrategy: contextStrategy.id, createdAt: new Date().toISOString(), sourceRevision: revision(cwd), productVersion: rootPackage.version, @@ -89,7 +101,8 @@ export async function experimentMain(args: string[], cwd = process.cwd()): Promi const day = new Date().toLocaleDateString("en-CA"); const output = resolve( cwd, - arg(args, "out") ?? join(".docdex", "evals", `${day}-decision-context-current`), + arg(args, "out") ?? + join(".docdex", "evals", `${day}-decision-current-context-${contextStrategy.id}`), ); mkdirSync(dirname(join(output, "x")), { recursive: true }); writeFileSync(join(output, "results.json"), `${JSON.stringify(results, null, 2)}\n`); diff --git a/packages/evals/src/experiment/contextStrategies.ts b/packages/evals/src/experiment/contextStrategies.ts new file mode 100644 index 0000000..86e44b0 --- /dev/null +++ b/packages/evals/src/experiment/contextStrategies.ts @@ -0,0 +1,69 @@ +import { buildDocumentContextIndex } from "@lucid/core"; +import { buildDocumentContext, buildOutline } from "@lucid/kit"; +import type { ContextStrategy, ContextStrategyResult } from "./types"; + +const planned = ( + text: string, + selection: ReturnType["selection"], +): ContextStrategyResult => ({ + text, + selectedIds: selection.selectedBlockIds, + scope: selection.scope, + missingRefs: selection.missingRefs, + ambiguityKinds: selection.ambiguities.map(({ kind }) => kind), +}); + +export const currentContextStrategy: ContextStrategy = { + id: "current", + select(testCase, doc) { + const result = buildDocumentContext({ + doc, + utterance: testCase.utterance, + cursorBlockId: testCase.cursorBlockId, + recentTouched: testCase.recentTouched, + }); + return planned(result.text, result.selection); + }, +}; + +/** + * Experiment-only R1 candidate. Resolution stays with Lucid's existing planner; once the target is + * unambiguous, an agent read contains the resolved blocks and their section headings rather than + * every block in those sections. + */ +export const focusedContextStrategy: ContextStrategy = { + id: "focused", + select(testCase, doc) { + const result = buildDocumentContext({ + doc, + utterance: testCase.utterance, + cursorBlockId: testCase.cursorBlockId, + recentTouched: testCase.recentTouched, + }); + const unresolved = + result.selection.missingRefs.length > 0 || result.selection.ambiguities.length > 0; + if (result.selection.scope === "document" || unresolved) + return planned(result.text, result.selection); + + const index = buildDocumentContextIndex(doc, testCase.recentTouched); + const include = new Set(result.selection.fullBlockIds); + for (const id of result.selection.fullBlockIds) { + const sectionId = index.blocks.find((block) => block.id === id)?.sectionId; + const headingId = index.sections.find((section) => section.id === sectionId)?.headingId; + if (headingId) include.add(headingId); + } + const selectedIds = index.blocks.filter(({ id }) => include.has(id)).map(({ id }) => id); + const outline = buildOutline(doc, { + include: selectedIds, + full: result.selection.fullBlockIds, + maxChars: 24_000, + }); + return { + text: outline.text, + selectedIds, + scope: result.selection.scope, + missingRefs: result.selection.missingRefs, + ambiguityKinds: [], + }; + }, +}; diff --git a/packages/evals/src/experiment/experiment.test.ts b/packages/evals/src/experiment/experiment.test.ts index f5a479a..56a963f 100644 --- a/packages/evals/src/experiment/experiment.test.ts +++ b/packages/evals/src/experiment/experiment.test.ts @@ -1,6 +1,7 @@ import { describe, expect, it } from "vitest"; import { currentBaseline } from "./baseline"; import { CONTEXT_CASES, ROUTING_CASES, validateExperimentCases } from "./cases"; +import { currentContextStrategy, focusedContextStrategy } from "./contextStrategies"; import { experimentToMarkdown, passesExperimentGates, @@ -54,13 +55,26 @@ describe("the current routing baseline", () => { describe("targeted agent context", () => { it("freezes the current planner's strengths and known over-selection", () => { - const trials = CONTEXT_CASES.map(runContextCase); + const trials = CONTEXT_CASES.map((testCase) => + runContextCase(testCase, currentContextStrategy), + ); const summary = summariseContext(trials); expect(summary.passRate).toBe(0.625); expect(summary.meanTargetRecall).toBe(1); expect(summary.forbiddenSelections).toBe(3); expect(summary.medianEligibleContextReduction).toBeGreaterThan(0.8); }); + + it("focused reads remove the baseline over-selection without losing targets", () => { + const trials = CONTEXT_CASES.map((testCase) => + runContextCase(testCase, focusedContextStrategy), + ); + const summary = summariseContext(trials); + expect(summary.passRate).toBe(1); + expect(summary.meanTargetRecall).toBe(1); + expect(summary.forbiddenSelections).toBe(0); + expect(summary.medianContextReduction).toBeGreaterThan(0.2); + }); }); describe("experiment reports", () => { @@ -69,6 +83,7 @@ describe("experiment reports", () => { meta: { datasetVersion: 1, candidate: "current", + contextStrategy: "current", createdAt: "2026-10-20T00:00:00.000Z", sourceRevision: "abc123", productVersion: "0.4.0", @@ -86,7 +101,7 @@ describe("experiment reports", () => { routing: await Promise.all( ROUTING_CASES.map((testCase) => runRoutingCase(testCase, currentBaseline)), ), - context: CONTEXT_CASES.map(runContextCase), + context: CONTEXT_CASES.map((testCase) => runContextCase(testCase, currentContextStrategy)), }; const markdown = experimentToMarkdown(results); expect(markdown).toContain("High-confidence wrong direct actions: 0"); diff --git a/packages/evals/src/experiment/report.ts b/packages/evals/src/experiment/report.ts index decf2c4..c84a041 100644 --- a/packages/evals/src/experiment/report.ts +++ b/packages/evals/src/experiment/report.ts @@ -93,6 +93,7 @@ export function experimentToMarkdown(results: ExperimentResults): string { `# Lucid decision/context experiment: ${results.meta.candidate}`, "", `- Dataset: v${String(results.meta.datasetVersion)} (${results.meta.split})`, + `- Routing/context candidates: ${results.meta.candidate} / ${results.meta.contextStrategy}`, `- Source: ${results.meta.sourceRevision} ยท Lucid ${results.meta.productVersion}`, `- System: ${results.meta.hardware.platform} ${results.meta.hardware.release} ${results.meta.hardware.arch}; ${results.meta.hardware.cpu}; ${String(results.meta.hardware.logicalCpus)} logical CPUs; ${(results.meta.hardware.totalMemoryBytes / 1024 ** 3).toFixed(1)} GiB`, `- Created: ${results.meta.createdAt}`, @@ -112,11 +113,11 @@ export function experimentToMarkdown(results: ExperimentResults): string { `- Median reduction on eligible large-context tasks: ${percent(context.medianEligibleContextReduction)}`, `- Forbidden block selections: ${String(context.forbiddenSelections)}`, "", - "| Case | Split | Recall | Reduction | Selected | Result |", - "|---|---|---:|---:|---:|---|", + "| Strategy | Case | Split | Recall | Reduction | Selected | Result |", + "|---|---|---|---:|---:|---:|---|", ...results.context.map( (trial) => - `| ${trial.caseId} | ${trial.split} | ${percent(trial.targetRecall)} | ${percent(trial.reductionRate)} | ${String(trial.selectedIds.length)} | ${trial.pass ? "pass" : "FAIL"} |`, + `| ${trial.strategy} | ${trial.caseId} | ${trial.split} | ${percent(trial.targetRecall)} | ${percent(trial.reductionRate)} | ${String(trial.selectedIds.length)} | ${trial.pass ? "pass" : "FAIL"} |`, ), ]; const failures = [ diff --git a/packages/evals/src/experiment/run.ts b/packages/evals/src/experiment/run.ts index 9e69201..8dcd726 100644 --- a/packages/evals/src/experiment/run.ts +++ b/packages/evals/src/experiment/run.ts @@ -1,8 +1,10 @@ -import { buildDocumentContext, buildOutline } from "@lucid/kit"; +import { buildOutline } from "@lucid/kit"; +import { currentContextStrategy } from "./contextStrategies"; import { contextFixture } from "./fixtures"; import type { ContextCase, ContextTrial, + ContextStrategy, RoutingCandidate, RoutingCase, RoutingTrial, @@ -54,32 +56,30 @@ export async function runRoutingCase( }; } -export function runContextCase(testCase: ContextCase): ContextTrial { +export function runContextCase( + testCase: ContextCase, + strategy: ContextStrategy = currentContextStrategy, +): ContextTrial { const doc = contextFixture(testCase.fixture); const allIds = doc.blocks.map((block) => block.id); const baseline = buildOutline(doc, { full: allIds, maxChars: 80_000 }); - const targeted = buildDocumentContext({ - doc, - utterance: testCase.utterance, - cursorBlockId: testCase.cursorBlockId, - recentTouched: testCase.recentTouched, - }); - const selected = targeted.selection.selectedBlockIds; + const targeted = strategy.select(testCase, doc); + const selected = targeted.selectedIds; const requiredHits = testCase.requiredIds.filter((id) => selected.includes(id)).length; const targetRecall = testCase.requiredIds.length === 0 ? 1 : requiredHits / testCase.requiredIds.length; const targetPrecision = selected.length === 0 ? 1 : requiredHits / selected.length; const forbiddenSelected = testCase.forbiddenIds.filter((id) => selected.includes(id)); - const missingRefs = targeted.selection.missingRefs; - const ambiguityKinds = targeted.selection.ambiguities.map(({ kind }) => kind); + const missingRefs = targeted.missingRefs; + const ambiguityKinds = targeted.ambiguityKinds; const expectedMissing = testCase.expectedMissingRefs ?? []; const expectedAmbiguity = testCase.expectedAmbiguityKinds ?? []; const failures: string[] = []; if (targetRecall !== 1) failures.push(`target recall was ${targetRecall.toFixed(3)}`); if (forbiddenSelected.length > 0) failures.push(`selected forbidden blocks: ${forbiddenSelected.join(", ")}`); - if (targeted.selection.scope !== testCase.expectedScope) - failures.push(`expected scope ${testCase.expectedScope}, got ${targeted.selection.scope}`); + if (targeted.scope !== testCase.expectedScope) + failures.push(`expected scope ${testCase.expectedScope}, got ${targeted.scope}`); if (!sameStrings(missingRefs, expectedMissing)) failures.push( `expected missing refs ${expectedMissing.join(", ") || "none"}, got ${missingRefs.join(", ") || "none"}`, @@ -89,6 +89,7 @@ export function runContextCase(testCase: ContextCase): ContextTrial { `expected ambiguities ${expectedAmbiguity.join(", ") || "none"}, got ${ambiguityKinds.join(", ") || "none"}`, ); return { + strategy: strategy.id, caseId: testCase.id, family: testCase.family, split: testCase.split, @@ -102,7 +103,7 @@ export function runContextCase(testCase: ContextCase): ContextTrial { targetRecall, targetPrecision, forbiddenSelected, - scope: targeted.selection.scope, + scope: targeted.scope, missingRefs, ambiguityKinds, pass: failures.length === 0, diff --git a/packages/evals/src/experiment/types.ts b/packages/evals/src/experiment/types.ts index 8fab456..ca03e54 100644 --- a/packages/evals/src/experiment/types.ts +++ b/packages/evals/src/experiment/types.ts @@ -60,6 +60,19 @@ export interface ContextCase { expectedAmbiguityKinds?: Array<"table" | "chart" | "diagram">; } +export interface ContextStrategyResult { + text: string; + selectedIds: string[]; + scope: "targeted" | "document"; + missingRefs: string[]; + ambiguityKinds: Array<"table" | "chart" | "diagram">; +} + +export interface ContextStrategy { + id: string; + select(testCase: ContextCase, doc: LucidDoc): ContextStrategyResult; +} + export interface RoutingTrial { caseId: string; family: string; @@ -73,6 +86,7 @@ export interface RoutingTrial { } export interface ContextTrial { + strategy: string; caseId: string; family: string; split: ExperimentSplit; @@ -96,6 +110,7 @@ export interface ContextTrial { export interface ExperimentMeta { datasetVersion: number; candidate: string; + contextStrategy: string; createdAt: string; sourceRevision: string; productVersion: string;