|
| 1 | +/** |
| 2 | + * @license |
| 3 | + * Copyright 2026 Google LLC |
| 4 | + * SPDX-License-Identifier: Apache-2.0 |
| 5 | + * |
| 6 | + * Eval scenario: user asks to fix issues with their webpage (no URL given). |
| 7 | + * When no URL is provided, the model should pick the current frontend and run |
| 8 | + * and inspect it. Verifies the MCP server is invoked and the model opens the |
| 9 | + * frontend and inspects it (snapshot, console, or network). |
| 10 | + * |
| 11 | + * Note: Tools like performance_start_trace, take_snapshot, list_console_messages, |
| 12 | + * and list_network_requests do not require a URL in the prompt—they operate on |
| 13 | + * the currently selected page. Only navigate_page/new_page need a URL to open |
| 14 | + * a page; the eval runner injects the test URL when htmlRoute is set. |
| 15 | + */ |
| 16 | + |
| 17 | +import assert from 'node:assert'; |
| 18 | + |
| 19 | +import type {TestScenario} from '../eval_gemini.ts'; |
| 20 | + |
| 21 | +const INSPECTION_TOOLS = [ |
| 22 | + 'take_snapshot', |
| 23 | + 'list_console_messages', |
| 24 | + 'list_network_requests', |
| 25 | +]; |
| 26 | + |
| 27 | +export const scenario: TestScenario = { |
| 28 | + prompt: 'Can you fix issues with my webpage?', |
| 29 | + maxTurns: 4, |
| 30 | + htmlRoute: { |
| 31 | + path: '/fix_issues_test.html', |
| 32 | + htmlContent: ` |
| 33 | + <h1>Test Page</h1> |
| 34 | + <p>Some content</p> |
| 35 | + <script> |
| 36 | + console.error('Intentional error for testing'); |
| 37 | + </script> |
| 38 | + `, |
| 39 | + }, |
| 40 | + expectations: calls => { |
| 41 | + const NAVIGATION_TOOLS = ['navigate_page', 'new_page']; |
| 42 | + assert.ok( |
| 43 | + calls.length >= 2, |
| 44 | + 'Expected at least navigation and one inspection', |
| 45 | + ); |
| 46 | + const navigationIndex = calls.findIndex(c => |
| 47 | + NAVIGATION_TOOLS.includes(c.name), |
| 48 | + ); |
| 49 | + assert.ok( |
| 50 | + navigationIndex !== -1, |
| 51 | + `Expected a navigation call (${NAVIGATION_TOOLS.join(' or ')}), got: ${calls.map(c => c.name).join(', ')}`, |
| 52 | + ); |
| 53 | + const afterNavigation = calls.slice(navigationIndex + 1); |
| 54 | + const inspectionCalls = afterNavigation.filter(c => |
| 55 | + INSPECTION_TOOLS.includes(c.name), |
| 56 | + ); |
| 57 | + assert.ok( |
| 58 | + inspectionCalls.length >= 1, |
| 59 | + `Expected at least one inspection tool (${INSPECTION_TOOLS.join(', ')}) after navigation, got: ${calls.map(c => c.name).join(', ')}`, |
| 60 | + ); |
| 61 | + }, |
| 62 | +}; |
0 commit comments