eval_gemini.ts

/**
 * @license
 * Copyright 2026 Google LLC
 * SPDX-License-Identifier: Apache-2.0
 */

import fs from 'node:fs';
import path from 'node:path';
import {pathToFileURL} from 'node:url';
import {parseArgs} from 'node:util';

import {GoogleGenAI, mcpToTool} from '@google/genai';
import {Client} from '@modelcontextprotocol/sdk/client/index.js';
import {StdioClientTransport} from '@modelcontextprotocol/sdk/client/stdio.js';

import {TestServer} from '../build/tests/server.js';

const ROOT_DIR = path.resolve(import.meta.dirname, '..');
const SCENARIOS_DIR = path.join(import.meta.dirname, 'eval_scenarios');
const SKILL_PATH = path.join(ROOT_DIR, 'skills', 'chrome-devtools', 'SKILL.md');

// Define schema for our test scenarios
export interface CapturedFunctionCall {
  name: string;
  args: Record<string, unknown>;
}

export interface TestScenario {
  prompt: string;
  maxTurns: number;
  expectations: (calls: CapturedFunctionCall[]) => void;
  htmlRoute?: {
    path: string;
    htmlContent: string;
  };
  /** Extra CLI flags passed to the MCP server (e.g. '--experimental-page-id-routing'). */
  serverArgs?: string[];
}

async function loadScenario(scenarioPath: string): Promise<TestScenario> {
  const module = await import(pathToFileURL(scenarioPath).href);
  if (!module.scenario) {
    throw new Error(
      `Scenario file ${scenarioPath} does not export a 'scenario' object.`,
    );
  }
  return module.scenario;
}

async function runSingleScenario(
  scenarioPath: string,
  apiKey: string,
  server: TestServer,
  modelId: string,
  debug: boolean,
  includeSkill: boolean,
): Promise<void> {
  const debugLog = (...args: unknown[]) => {
    if (debug) {
      console.log(...args);
    }
  };
  const absolutePath = path.resolve(scenarioPath);
  debugLog(
    `\n### Running Scenario: ${path.relative(ROOT_DIR, absolutePath)} ###`,
  );

  let client: Client | undefined;
  let transport: StdioClientTransport | undefined;

  try {
    const loadedScenario = await loadScenario(absolutePath);
    const scenario = {...loadedScenario};

    // Prepend skill content if requested
    if (includeSkill) {
      if (!fs.existsSync(SKILL_PATH)) {
        throw new Error(
          `Skill file not found at ${SKILL_PATH}. Please ensure the skill file exists.`,
        );
      }
      const skillContent = fs.readFileSync(SKILL_PATH, 'utf-8');
      scenario.prompt = `${skillContent}\n\n---\n\n${scenario.prompt}`;
    }

    // Append random queryid to avoid caching issues and test distinct runs
    const randomId = Math.floor(Math.random() * 1000000);
    scenario.prompt = `${scenario.prompt}\nqueryid=${randomId}`;

    if (scenario.htmlRoute) {
      server.addHtmlRoute(
        scenario.htmlRoute.path,
        scenario.htmlRoute.htmlContent,
      );
      scenario.prompt = scenario.prompt.replace(
        '<TEST_URL>',
        server.getRoute(scenario.htmlRoute.path),
      );
    }

    // Path to the compiled MCP server
    const serverPath = path.join(
      ROOT_DIR,
      'build/src/bin/chrome-devtools-mcp.js',
    );
    if (!fs.existsSync(serverPath)) {
      throw new Error(
        `MCP server not found at ${serverPath}. Please run 'npm run build' first.`,
      );
    }

    // Environment variables
    const env: Record<string, string> = {};
    Object.entries(process.env).forEach(([key, value]) => {
      if (value !== undefined) {
        env[key] = value;
      }
    });
    env['CHROME_DEVTOOLS_MCP_NO_USAGE_STATISTICS'] = 'true';

    const args = [serverPath];
    if (!debug) {
      args.push('--headless');
    }
    if (scenario.serverArgs) {
      args.push(...scenario.serverArgs);
    }

    transport = new StdioClientTransport({
      command: 'node',
      args,
      env,
      stderr: debug ? 'inherit' : 'ignore',
    });

    client = new Client(
      {name: 'gemini-eval-client', version: '1.0.0'},
      {capabilities: {}},
    );

    await client.connect(transport);

    const allCalls: CapturedFunctionCall[] = [];
    const originalCallTool = client.callTool.bind(client);
    client.callTool = async (request, schema) => {
      // NOTE: request.name is the original name as the MCP client sees it.
      // mcpToTool handles the conversion from Gemini sanitized name to original name.
      debugLog(
        `Executing tool: ${request.name} with args: ${JSON.stringify(request.arguments)}`,
      );
      allCalls.push({
        name: request.name,
        args: (request.arguments as Record<string, unknown>) || {},
      });
      const response = await originalCallTool(request, schema);
      debugLog(`Tool response: ${JSON.stringify(response)}`);
      return response;
    };

    const ai = new GoogleGenAI({apiKey});

    debugLog(`\n--- Prompt ---\n${scenario.prompt}`);

    const result = await ai.models.generateContent({
      model: modelId,
      contents: scenario.prompt,
      config: {
        tools: [mcpToTool(client)],
        automaticFunctionCalling: {
          maximumRemoteCalls: scenario.maxTurns,
        },
      },
    });

    debugLog(`\n--- Response ---\n${result.text}`);

    debugLog('\nVerifying expectations...');
    scenario.expectations(allCalls);
  } finally {
    try {
      await client?.close();
    } catch (e) {
      console.error('Error closing client:', e);
    }
  }
}

async function main() {
  const apiKey = process.env.GEMINI_API_KEY;
  if (!apiKey) {
    throw new Error('GEMINI_API_KEY environment variable is required.');
  }

  const {values, positionals} = parseArgs({
    options: {
      model: {
        type: 'string',
        default: 'gemini-2.5-flash',
      },
      debug: {
        type: 'boolean',
        default: false,
      },
      repeat: {
        type: 'boolean',
        default: false,
      },
      'include-skill': {
        type: 'boolean',
        default: false,
      },
    },
    allowPositionals: true,
  });

  const modelId = values.model;
  const debug = values.debug;
  const repeat = values.repeat;
  const includeSkill = values['include-skill'];

  const scenarioFiles =
    positionals.length > 0
      ? positionals.map(p => path.resolve(p))
      : fs
          .readdirSync(SCENARIOS_DIR)
          .filter(file => file.endsWith('.ts') || file.endsWith('.js'))
          .map(file => path.join(SCENARIOS_DIR, file));

  const server = new TestServer(TestServer.randomPort());
  await server.start();

  let successCount = 0;
  let failureCount = 0;

  try {
    for (const scenarioPath of scenarioFiles) {
      for (let i = 1; i <= (repeat ? 3 : 1); i++) {
        try {
          if (debug) {
            console.log(
              `Running scenario: ${path.relative(ROOT_DIR, scenarioPath)} (Run ${i}/3)`,
            );
          }
          await runSingleScenario(
            scenarioPath,
            apiKey,
            server,
            modelId,
            debug,
            includeSkill,
          );
          console.log(`✔ ${path.relative(ROOT_DIR, scenarioPath)} (Run ${i})`);
          successCount++;
        } catch (e) {
          console.error(
            `✖ ${path.relative(ROOT_DIR, scenarioPath)} (Run ${i})`,
          );
          console.error(e);
          failureCount++;
        } finally {
          server.restore();
        }
      }
    }
  } finally {
    await server.stop();
  }

  console.log(`\nSummary: ${successCount} passed, ${failureCount} failed`);

  if (failureCount > 0) {
    process.exit(1);
  }
}

main().catch(error => {
  console.error('Fatal error:', error);
  process.exit(1);
});