chore: support an html route in evals

Author: OrKoN

scripts/eval_gemini.ts

@@ -18,6 +18,7 @@ import {StdioClientTransport} from '@modelcontextprotocol/sdk/client/stdio.js';
 
 const ROOT_DIR = path.resolve(import.meta.dirname, '..');
 const SCENARIOS_DIR = path.join(import.meta.dirname, 'eval_scenarios');
+import {TestServer} from '../build/tests/server.js';
 
 // Define schema for our test scenarios
 export interface CapturedFunctionCall {
@@ -29,6 +30,10 @@ export interface TestScenario {
   prompt: string;
   maxTurns: number;
   expectations: (calls: CapturedFunctionCall[]) => void;
+  htmlRoute?: {
+    path: string;
+    htmlContent: string;
+  };
 }
 
 async function loadScenario(scenarioPath: string): Promise<TestScenario> {
@@ -84,10 +89,24 @@ async function runSingleScenario(
 
   let client: Client | undefined;
   let transport: StdioClientTransport | undefined;
+  let server: TestServer | undefined;
 
   try {
     const scenario = await loadScenario(absolutePath);
 
+    if (scenario.htmlRoute) {
+      server = new TestServer(TestServer.randomPort());
+      await server.start();
+      server.addHtmlRoute(
+        scenario.htmlRoute.path,
+        scenario.htmlRoute.htmlContent,
+      );
+      scenario.prompt = scenario.prompt.replace(
+        '<TEST_URL>',
+        server.getRoute(scenario.htmlRoute.path),
+      );
+    }
+
     // Path to the compiled MCP server
     const serverPath = path.join(ROOT_DIR, 'build/src/index.js');
     if (!fs.existsSync(serverPath)) {
@@ -148,7 +167,7 @@ async function runSingleScenario(
 
     const genAI = new GoogleGenerativeAI(apiKey);
     const model = genAI.getGenerativeModel({
-      model: 'gemini-3-pro-preview',
+      model: 'gemini-2.5-flash',
       tools: [{functionDeclarations}],
     });
 
@@ -167,7 +186,9 @@ async function runSingleScenario(
     console.log(`\n--- Turn 1 (User) ---`);
     console.log(scenario.prompt);
 
-    let result = await chat.sendMessage(scenario.prompt);
+    let result = await chat.sendMessage(scenario.prompt, {
+      timeout: 5000,
+    });
     let response = result.response;
 
     while (turnCount < scenario.maxTurns) {
@@ -248,6 +269,7 @@ async function runSingleScenario(
   } finally {
     await client?.close();
     await transport?.close();
+    await server?.stop();
   }
 }
 const apiKey = process.env.GEMINI_API_KEY;

scripts/eval_scenarios/snapshot_test.ts

@@ -0,0 +1,24 @@
+/**
+ * @license
+ * Copyright 2026 Google LLC
+ * SPDX-License-Identifier: Apache-2.0
+ */
+import assert from 'node:assert';
+
+import type {TestScenario} from '../eval_gemini.ts';
+
+export const scenario: TestScenario = {
+  prompt: 'Read the content of <TEST_URL>',
+  maxTurns: 3,
+  htmlRoute: {
+    path: '/test.html',
+    htmlContent: '<h1>Hello World</h1><p>This is a test.</p>',
+  },
+  expectations: calls => {
+    assert.strictEqual(calls.length, 2);
+    assert.ok(
+      calls[0].name === 'navigate_page' || calls[0].name === 'new_page',
+    );
+    assert.ok(calls[1].name === 'take_snapshot');
+  },
+};

tests/server.ts

@@ -13,7 +13,7 @@ import {before, after, afterEach} from 'node:test';
 
 import {html} from './utils.js';
 
-class TestServer {
+export class TestServer {
   #port: number;
   #server: Server;