From 63f58d3bbd2a595f3ffa04a769105408f539c0f5 Mon Sep 17 00:00:00 2001 From: Ben Vinegar Date: Thu, 2 Jul 2026 02:03:49 -0400 Subject: [PATCH] test(mcp): guard surface schema drift --- .changeset/curvy-parks-fall.md | 2 + test/mcpSpec.test.ts | 69 ++++++++++++++++++++++++++-------- 2 files changed, 56 insertions(+), 15 deletions(-) create mode 100644 .changeset/curvy-parks-fall.md diff --git a/.changeset/curvy-parks-fall.md b/.changeset/curvy-parks-fall.md new file mode 100644 index 0000000..a845151 --- /dev/null +++ b/.changeset/curvy-parks-fall.md @@ -0,0 +1,2 @@ +--- +--- diff --git a/test/mcpSpec.test.ts b/test/mcpSpec.test.ts index 3f11a6a..6bf2f99 100644 --- a/test/mcpSpec.test.ts +++ b/test/mcpSpec.test.ts @@ -21,43 +21,82 @@ import { // enum, the stdio zod enum, and the runtime validator. Add a kind to types.ts // without teaching MCP about it (or the validator) and one of these fails. -// The exact `kind` enum a client receives for the canonical publish tool from -// the HTTP `tools/list` response. -const httpKindEnum = (() => { +// The exact surface JSON Schema a client receives for the canonical publish +// tool from the HTTP `tools/list` response. +const httpSurfaceSchema = (() => { const tool = HTTP_MCP_TOOLS.find((t) => t.name === "publish_post"); assert.ok(tool, "publish_post tool must exist"); + return (tool as any).inputSchema.properties.surfaces.items; +})(); + +const httpKindEnum = (() => { // inputSchema.properties.surfaces.items.properties.kind.enum — the wire path. - const enumValues = (tool as any).inputSchema.properties.surfaces.items.properties.kind.enum; + const enumValues = httpSurfaceSchema.properties.kind.enum; assert.ok(Array.isArray(enumValues), "surfaces.items.kind.enum must be an array"); return enumValues as string[]; })(); -// A minimal valid example per kind, used to prove the schema + validator accept -// each one. Kept deliberately minimal so a field going missing from the MCP -// schema surfaces here. +// A representative valid example per kind, used to prove the schema + +// validator accept each advertised payload field. Include optional fields too so +// a field going missing from the MCP schema surfaces here. const EXAMPLES: Record<(typeof SURFACE_KINDS)[number], Surface> = { - html: { kind: "html", html: "

hi

" }, - diff: { kind: "diff", files: [{ filename: "a.ts", before: "a", after: "b" }] }, - image: { kind: "image", assetId: "asset123" }, - trace: { kind: "trace", steps: [{ label: "step one" }] }, + html: { kind: "html", html: "

hi

", kits: ["issues"] }, + diff: { + kind: "diff", + patch: "--- a/x\n+++ b/x\n@@ -1 +1 @@\n-a\n+b", + files: [{ filename: "a.ts", before: "a", after: "b", language: "ts" }], + layout: "split", + }, + image: { kind: "image", assetId: "asset123", alt: "screenshot", caption: "after" }, + trace: { + kind: "trace", + assetId: "trace123", + title: "Run trace", + steps: [{ label: "step one", kind: "tool", detail: "ok", ts: "2026-07-02T00:00:00Z" }], + }, markdown: { kind: "markdown", markdown: "# heading" }, - terminal: { kind: "terminal", text: "$ ls\nfile.txt" }, + terminal: { kind: "terminal", text: "$ ls\nfile.txt", cols: 80, title: "shell" }, mermaid: { kind: "mermaid", mermaid: "flowchart TD\nA-->B" }, json: { kind: "json", data: { ok: true, items: [1, 2, 3] } }, - code: { kind: "code", code: "const x = 1;", language: "ts" }, + code: { kind: "code", code: "const x = 1;", language: "ts", title: "x.ts", lineStart: 10 }, }; test("HTTP publish_post advertises exactly the canonical kind set", () => { assert.deepEqual([...httpKindEnum].sort(), [...SURFACE_KINDS].sort()); }); +test("HTTP publish_post advertises every field used by canonical examples", () => { + const assertFieldsAdvertised = (schema: any, value: object, path: string) => { + assert.ok(schema?.properties, `${path} must advertise object properties`); + for (const [key, nested] of Object.entries(value)) { + assert.ok(Object.hasOwn(schema.properties, key), `${path} must advertise ${key}`); + if ( + Array.isArray(nested) && + nested.length > 0 && + typeof nested[0] === "object" && + nested[0] !== null + ) { + assertFieldsAdvertised( + schema.properties[key].items, + nested[0] as object, + `${path}.${key}[]`, + ); + } + } + }; + + for (const kind of SURFACE_KINDS) { + assertFieldsAdvertised(httpSurfaceSchema, EXAMPLES[kind], `surface ${kind}`); + } +}); + test("every canonical kind has a worked example (no kind left untested)", () => { for (const kind of SURFACE_KINDS) { assert.ok(EXAMPLES[kind], `missing test example for kind "${kind}"`); } }); -test("the stdio publish schema accepts a minimal example of every kind", () => { +test("the stdio publish schema accepts a representative example of every kind", () => { // The stdio schema object is z.object(STDIO_MCP_INPUT_SCHEMAS.publishPost); // its `surfaces` field is the array schema the MCP SDK enforces. const publishSchema = z.object(STDIO_MCP_INPUT_SCHEMAS.publishPost); @@ -76,7 +115,7 @@ test("the stdio publish schema rejects an unknown kind", () => { assert.equal(result.success, false); }); -test("the runtime validator accepts a minimal example of every kind", async () => { +test("the runtime validator accepts a representative example of every kind", async () => { for (const kind of SURFACE_KINDS) { const result = await validateSurfaces([EXAMPLES[kind]]); assert.ok(result.ok, `validator rejected kind "${kind}": ${result.ok ? "" : result.error}`);