Evaluate focused agent context against the v0.4 baseline

docs/evals.md

@@ -36,9 +36,11 @@ The first candidate (`current`) freezes v0.4.0's exact meta commands, conservati
 context as the baseline. Cases are grouped by intent family into development, calibration, and holdout splits so paraphrases
 cannot leak between them.
 
-Use `--suite routing|context|all`, `--split development|calibration|holdout|all`, or `--out <dir>`. By default the JSON and
-Markdown reports are written under `.docdex/evals/`, which is local experiment state and must not be committed. Candidate
-models remain outside the product runtime until they beat this baseline without a safety or final-task regression.
+Use `--suite routing|context|all`, `--split development|calibration|holdout|all`, `--context current|focused`, or `--out <dir>`.
+The `focused` context candidate is experiment-only: it keeps Lucid's target resolution but limits resolved reads to the target
+and its section heading. By default JSON and Markdown reports are written under `.docdex/evals/`, which is local experiment
+state and must not be committed. Candidates remain outside the product runtime until they beat the baseline without a safety
+or final-task regression.
 
 ## The bars
 

packages/evals/src/experiment/cases.ts

@@ -360,7 +360,7 @@ export const CONTEXT_CASES: ContextCase[] = [
     split: "holdout",
     fixture: "large-sections",
     utterance: "change Quasar retention to one hundred days",
-    requiredIds: sectionIds(6),
+    requiredIds: ["section-6", "topic-6-4"],
     forbiddenIds: otherSections(6),
     expectedScope: "targeted",
   },
@@ -371,7 +371,7 @@ export const CONTEXT_CASES: ContextCase[] = [
     fixture: "large-sections",
     utterance: "make this clearer",
     cursorBlockId: "topic-3-4",
-    requiredIds: sectionIds(3),
+    requiredIds: ["section-3", "topic-3-4"],
     forbiddenIds: otherSections(3),
     expectedScope: "targeted",
   },

packages/evals/src/experiment/cli.ts

@@ -4,6 +4,7 @@ import { arch, cpus, platform, release, totalmem } from "node:os";
 import { dirname, join, resolve } from "node:path";
 import { currentBaseline } from "./baseline";
 import { CONTEXT_CASES, ROUTING_CASES, validateExperimentCases } from "./cases";
+import { currentContextStrategy, focusedContextStrategy } from "./contextStrategies";
 import { experimentToMarkdown, passesExperimentGates } from "./report";
 import { runContextCase, runRoutingCase } from "./run";
 import { EXPERIMENT_DATASET_VERSION, type ExperimentResults, type ExperimentSplit } from "./types";
@@ -29,6 +30,7 @@ export async function experimentMain(args: string[], cwd = process.cwd()): Promi
   const candidateArg = arg(args, "candidate") ?? "current";
   const suite = arg(args, "suite") ?? "all";
   const splitArg = arg(args, "split") ?? "all";
+  const contextArg = arg(args, "context") ?? "current";
   if (candidateArg !== "current") {
     console.error("Only --candidate current exists in the baseline slice.");
     return 2;
@@ -41,6 +43,10 @@ export async function experimentMain(args: string[], cwd = process.cwd()): Promi
     console.error("--split must be development, calibration, holdout, or all.");
     return 2;
   }
+  if (contextArg !== "current" && contextArg !== "focused") {
+    console.error("--context must be current or focused.");
+    return 2;
+  }
   const problems = validateExperimentCases();
   if (problems.length > 0) {
     console.error(
@@ -60,7 +66,12 @@ export async function experimentMain(args: string[], cwd = process.cwd()): Promi
             runRoutingCase(testCase, currentBaseline),
           ),
         );
-  const context = suite === "routing" ? [] : CONTEXT_CASES.filter(inSplit).map(runContextCase);
+  const contextStrategy =
+    contextArg === "focused" ? focusedContextStrategy : currentContextStrategy;
+  const context =
+    suite === "routing"
+      ? []
+      : CONTEXT_CASES.filter(inSplit).map((testCase) => runContextCase(testCase, contextStrategy));
   const rootPackage = JSON.parse(readFileSync(join(cwd, "package.json"), "utf8")) as {
     version: string;
   };
@@ -69,6 +80,7 @@ export async function experimentMain(args: string[], cwd = process.cwd()): Promi
     meta: {
       datasetVersion: EXPERIMENT_DATASET_VERSION,
       candidate: currentBaseline.id,
+      contextStrategy: contextStrategy.id,
       createdAt: new Date().toISOString(),
       sourceRevision: revision(cwd),
       productVersion: rootPackage.version,
@@ -89,7 +101,8 @@ export async function experimentMain(args: string[], cwd = process.cwd()): Promi
   const day = new Date().toLocaleDateString("en-CA");
   const output = resolve(
     cwd,
-    arg(args, "out") ?? join(".docdex", "evals", `${day}-decision-context-current`),
+    arg(args, "out") ??
+      join(".docdex", "evals", `${day}-decision-current-context-${contextStrategy.id}`),
   );
   mkdirSync(dirname(join(output, "x")), { recursive: true });
   writeFileSync(join(output, "results.json"), `${JSON.stringify(results, null, 2)}\n`);

packages/evals/src/experiment/contextStrategies.ts

@@ -0,0 +1,69 @@
+import { buildDocumentContextIndex } from "@lucid/core";
+import { buildDocumentContext, buildOutline } from "@lucid/kit";
+import type { ContextStrategy, ContextStrategyResult } from "./types";
+
+const planned = (
+  text: string,
+  selection: ReturnType<typeof buildDocumentContext>["selection"],
+): ContextStrategyResult => ({
+  text,
+  selectedIds: selection.selectedBlockIds,
+  scope: selection.scope,
+  missingRefs: selection.missingRefs,
+  ambiguityKinds: selection.ambiguities.map(({ kind }) => kind),
+});
+
+export const currentContextStrategy: ContextStrategy = {
+  id: "current",
+  select(testCase, doc) {
+    const result = buildDocumentContext({
+      doc,
+      utterance: testCase.utterance,
+      cursorBlockId: testCase.cursorBlockId,
+      recentTouched: testCase.recentTouched,
+    });
+    return planned(result.text, result.selection);
+  },
+};
+
+/**
+ * Experiment-only R1 candidate. Resolution stays with Lucid's existing planner; once the target is
+ * unambiguous, an agent read contains the resolved blocks and their section headings rather than
+ * every block in those sections.
+ */
+export const focusedContextStrategy: ContextStrategy = {
+  id: "focused",
+  select(testCase, doc) {
+    const result = buildDocumentContext({
+      doc,
+      utterance: testCase.utterance,
+      cursorBlockId: testCase.cursorBlockId,
+      recentTouched: testCase.recentTouched,
+    });
+    const unresolved =
+      result.selection.missingRefs.length > 0 || result.selection.ambiguities.length > 0;
+    if (result.selection.scope === "document" || unresolved)
+      return planned(result.text, result.selection);
+
+    const index = buildDocumentContextIndex(doc, testCase.recentTouched);
+    const include = new Set(result.selection.fullBlockIds);
+    for (const id of result.selection.fullBlockIds) {
+      const sectionId = index.blocks.find((block) => block.id === id)?.sectionId;
+      const headingId = index.sections.find((section) => section.id === sectionId)?.headingId;
+      if (headingId) include.add(headingId);
+    }
+    const selectedIds = index.blocks.filter(({ id }) => include.has(id)).map(({ id }) => id);
+    const outline = buildOutline(doc, {
+      include: selectedIds,
+      full: result.selection.fullBlockIds,
+      maxChars: 24_000,
+    });
+    return {
+      text: outline.text,
+      selectedIds,
+      scope: result.selection.scope,
+      missingRefs: result.selection.missingRefs,
+      ambiguityKinds: [],
+    };
+  },
+};

packages/evals/src/experiment/experiment.test.ts

@@ -1,6 +1,7 @@
 import { describe, expect, it } from "vitest";
 import { currentBaseline } from "./baseline";
 import { CONTEXT_CASES, ROUTING_CASES, validateExperimentCases } from "./cases";
+import { currentContextStrategy, focusedContextStrategy } from "./contextStrategies";
 import {
   experimentToMarkdown,
   passesExperimentGates,
@@ -54,13 +55,26 @@ describe("the current routing baseline", () => {
 
 describe("targeted agent context", () => {
   it("freezes the current planner's strengths and known over-selection", () => {
-    const trials = CONTEXT_CASES.map(runContextCase);
+    const trials = CONTEXT_CASES.map((testCase) =>
+      runContextCase(testCase, currentContextStrategy),
+    );
     const summary = summariseContext(trials);
     expect(summary.passRate).toBe(0.625);
     expect(summary.meanTargetRecall).toBe(1);
     expect(summary.forbiddenSelections).toBe(3);
     expect(summary.medianEligibleContextReduction).toBeGreaterThan(0.8);
   });
+
+  it("focused reads remove the baseline over-selection without losing targets", () => {
+    const trials = CONTEXT_CASES.map((testCase) =>
+      runContextCase(testCase, focusedContextStrategy),
+    );
+    const summary = summariseContext(trials);
+    expect(summary.passRate).toBe(1);
+    expect(summary.meanTargetRecall).toBe(1);
+    expect(summary.forbiddenSelections).toBe(0);
+    expect(summary.medianContextReduction).toBeGreaterThan(0.2);
+  });
 });
 
 describe("experiment reports", () => {
@@ -69,6 +83,7 @@ describe("experiment reports", () => {
       meta: {
         datasetVersion: 1,
         candidate: "current",
+        contextStrategy: "current",
         createdAt: "2026-10-20T00:00:00.000Z",
         sourceRevision: "abc123",
         productVersion: "0.4.0",
@@ -86,7 +101,7 @@ describe("experiment reports", () => {
       routing: await Promise.all(
         ROUTING_CASES.map((testCase) => runRoutingCase(testCase, currentBaseline)),
       ),
-      context: CONTEXT_CASES.map(runContextCase),
+      context: CONTEXT_CASES.map((testCase) => runContextCase(testCase, currentContextStrategy)),
     };
     const markdown = experimentToMarkdown(results);
     expect(markdown).toContain("High-confidence wrong direct actions: 0");

packages/evals/src/experiment/report.ts

@@ -93,6 +93,7 @@ export function experimentToMarkdown(results: ExperimentResults): string {
     `# Lucid decision/context experiment: ${results.meta.candidate}`,
     "",
     `- Dataset: v${String(results.meta.datasetVersion)} (${results.meta.split})`,
+    `- Routing/context candidates: ${results.meta.candidate} / ${results.meta.contextStrategy}`,
     `- Source: ${results.meta.sourceRevision} · Lucid ${results.meta.productVersion}`,
     `- System: ${results.meta.hardware.platform} ${results.meta.hardware.release} ${results.meta.hardware.arch}; ${results.meta.hardware.cpu}; ${String(results.meta.hardware.logicalCpus)} logical CPUs; ${(results.meta.hardware.totalMemoryBytes / 1024 ** 3).toFixed(1)} GiB`,
     `- Created: ${results.meta.createdAt}`,
@@ -112,11 +113,11 @@ export function experimentToMarkdown(results: ExperimentResults): string {
     `- Median reduction on eligible large-context tasks: ${percent(context.medianEligibleContextReduction)}`,
     `- Forbidden block selections: ${String(context.forbiddenSelections)}`,
     "",
-    "| Case | Split | Recall | Reduction | Selected | Result |",
-    "|---|---|---:|---:|---:|---|",
+    "| Strategy | Case | Split | Recall | Reduction | Selected | Result |",
+    "|---|---|---|---:|---:|---:|---|",
     ...results.context.map(
       (trial) =>
-        `| ${trial.caseId} | ${trial.split} | ${percent(trial.targetRecall)} | ${percent(trial.reductionRate)} | ${String(trial.selectedIds.length)} | ${trial.pass ? "pass" : "FAIL"} |`,
+        `| ${trial.strategy} | ${trial.caseId} | ${trial.split} | ${percent(trial.targetRecall)} | ${percent(trial.reductionRate)} | ${String(trial.selectedIds.length)} | ${trial.pass ? "pass" : "FAIL"} |`,
     ),
   ];
   const failures = [

packages/evals/src/experiment/run.ts

@@ -1,8 +1,10 @@
-import { buildDocumentContext, buildOutline } from "@lucid/kit";
+import { buildOutline } from "@lucid/kit";
+import { currentContextStrategy } from "./contextStrategies";
 import { contextFixture } from "./fixtures";
 import type {
   ContextCase,
   ContextTrial,
+  ContextStrategy,
   RoutingCandidate,
   RoutingCase,
   RoutingTrial,
@@ -54,32 +56,30 @@ export async function runRoutingCase(
   };
 }
 
-export function runContextCase(testCase: ContextCase): ContextTrial {
+export function runContextCase(
+  testCase: ContextCase,
+  strategy: ContextStrategy = currentContextStrategy,
+): ContextTrial {
   const doc = contextFixture(testCase.fixture);
   const allIds = doc.blocks.map((block) => block.id);
   const baseline = buildOutline(doc, { full: allIds, maxChars: 80_000 });
-  const targeted = buildDocumentContext({
-    doc,
-    utterance: testCase.utterance,
-    cursorBlockId: testCase.cursorBlockId,
-    recentTouched: testCase.recentTouched,
-  });
-  const selected = targeted.selection.selectedBlockIds;
+  const targeted = strategy.select(testCase, doc);
+  const selected = targeted.selectedIds;
   const requiredHits = testCase.requiredIds.filter((id) => selected.includes(id)).length;
   const targetRecall =
     testCase.requiredIds.length === 0 ? 1 : requiredHits / testCase.requiredIds.length;
   const targetPrecision = selected.length === 0 ? 1 : requiredHits / selected.length;
   const forbiddenSelected = testCase.forbiddenIds.filter((id) => selected.includes(id));
-  const missingRefs = targeted.selection.missingRefs;
-  const ambiguityKinds = targeted.selection.ambiguities.map(({ kind }) => kind);
+  const missingRefs = targeted.missingRefs;
+  const ambiguityKinds = targeted.ambiguityKinds;
   const expectedMissing = testCase.expectedMissingRefs ?? [];
   const expectedAmbiguity = testCase.expectedAmbiguityKinds ?? [];
   const failures: string[] = [];
   if (targetRecall !== 1) failures.push(`target recall was ${targetRecall.toFixed(3)}`);
   if (forbiddenSelected.length > 0)
     failures.push(`selected forbidden blocks: ${forbiddenSelected.join(", ")}`);
-  if (targeted.selection.scope !== testCase.expectedScope)
-    failures.push(`expected scope ${testCase.expectedScope}, got ${targeted.selection.scope}`);
+  if (targeted.scope !== testCase.expectedScope)
+    failures.push(`expected scope ${testCase.expectedScope}, got ${targeted.scope}`);
   if (!sameStrings(missingRefs, expectedMissing))
     failures.push(
       `expected missing refs ${expectedMissing.join(", ") || "none"}, got ${missingRefs.join(", ") || "none"}`,
@@ -89,6 +89,7 @@ export function runContextCase(testCase: ContextCase): ContextTrial {
       `expected ambiguities ${expectedAmbiguity.join(", ") || "none"}, got ${ambiguityKinds.join(", ") || "none"}`,
     );
   return {
+    strategy: strategy.id,
     caseId: testCase.id,
     family: testCase.family,
     split: testCase.split,
@@ -102,7 +103,7 @@ export function runContextCase(testCase: ContextCase): ContextTrial {
     targetRecall,
     targetPrecision,
     forbiddenSelected,
-    scope: targeted.selection.scope,
+    scope: targeted.scope,
     missingRefs,
     ambiguityKinds,
     pass: failures.length === 0,

packages/evals/src/experiment/types.ts

@@ -60,6 +60,19 @@ export interface ContextCase {
   expectedAmbiguityKinds?: Array<"table" | "chart" | "diagram">;
 }
 
+export interface ContextStrategyResult {
+  text: string;
+  selectedIds: string[];
+  scope: "targeted" | "document";
+  missingRefs: string[];
+  ambiguityKinds: Array<"table" | "chart" | "diagram">;
+}
+
+export interface ContextStrategy {
+  id: string;
+  select(testCase: ContextCase, doc: LucidDoc): ContextStrategyResult;
+}
+
 export interface RoutingTrial {
   caseId: string;
   family: string;
@@ -73,6 +86,7 @@ export interface RoutingTrial {
 }
 
 export interface ContextTrial {
+  strategy: string;
   caseId: string;
   family: string;
   split: ExperimentSplit;
@@ -96,6 +110,7 @@ export interface ContextTrial {
 export interface ExperimentMeta {
   datasetVersion: number;
   candidate: string;
+  contextStrategy: string;
   createdAt: string;
   sourceRevision: string;
   productVersion: string;