diff --git a/src/McpResponse.ts b/src/McpResponse.ts index ed20e4c4f..77898b5df 100644 --- a/src/McpResponse.ts +++ b/src/McpResponse.ts @@ -19,6 +19,7 @@ import type { Page, ResourceType, TextContent, + JSONSchema7Definition, } from './third_party/index.js'; import type {ToolGroup, ToolDefinition} from './tools/inPage.js'; import {handleDialog} from './tools/pages.js'; @@ -41,6 +42,57 @@ interface TraceInsightData { insightName: InsightName; } +export function replaceHtmlElementsWithUids(schema: JSONSchema7Definition) { + if (typeof schema === 'boolean') { + return; + } + + let isHtmlElement = false; + for (const [key, value] of Object.entries(schema)) { + if (key === 'x-mcp-type' && value === 'HTMLElement') { + isHtmlElement = true; + break; + } + } + + if (isHtmlElement) { + schema.properties = {uid: {type: 'string'}}; + schema.required = ['uid']; + } + + if (schema.properties) { + for (const key of Object.keys(schema.properties)) { + replaceHtmlElementsWithUids(schema.properties[key]); + } + } + + if (schema.items) { + if (Array.isArray(schema.items)) { + for (const item of schema.items) { + replaceHtmlElementsWithUids(item); + } + } else { + replaceHtmlElementsWithUids(schema.items); + } + } + + if (schema.anyOf) { + for (const s of schema.anyOf) { + replaceHtmlElementsWithUids(s); + } + } + if (schema.allOf) { + for (const s of schema.allOf) { + replaceHtmlElementsWithUids(s); + } + } + if (schema.oneOf) { + for (const s of schema.oneOf) { + replaceHtmlElementsWithUids(s); + } + } +} + async function getToolGroup( page: McpPage, ): Promise | undefined> { @@ -91,6 +143,10 @@ async function getToolGroup( }, 0); }); }); + + for (const tool of toolGroup?.tools ?? []) { + replaceHtmlElementsWithUids(tool.inputSchema); + } return toolGroup; } diff --git a/src/third_party/index.ts b/src/third_party/index.ts index 96f5295c0..09b0d02ce 100644 --- a/src/third_party/index.ts +++ b/src/third_party/index.ts @@ -41,7 +41,7 @@ export {default as puppeteer} from 'puppeteer-core'; export type * from 'puppeteer-core'; export {PipeTransport} from 'puppeteer-core/internal/node/PipeTransport.js'; export type {CdpPage} from 'puppeteer-core/internal/cdp/Page.js'; -export type {JSONSchema7} from 'json-schema'; +export type {JSONSchema7, JSONSchema7Definition} from 'json-schema'; export { resolveDefaultUserDataDir, detectBrowserPlatform, diff --git a/src/tools/inPage.ts b/src/tools/inPage.ts index 5b0031f62..0cb1b6ba7 100644 --- a/src/tools/inPage.ts +++ b/src/tools/inPage.ts @@ -4,7 +4,12 @@ * SPDX-License-Identifier: Apache-2.0 */ -import {zod, ajv, type JSONSchema7} from '../third_party/index.js'; +import { + zod, + ajv, + type JSONSchema7, + type ElementHandle, +} from '../third_party/index.js'; import {ToolCategory} from './categories.js'; import {definePageTool} from './ToolDefinition.js'; @@ -87,6 +92,22 @@ export const executeInPageTool = definePageTool({ } } + // Creates array of ElementHandles from the UIDs in the params. + // We do not replace the uids with the ElementsHandles yet, because + // the `evaluate` function only turns them into DOM elements if they + // are passed as non-nested arguments. + const handles: ElementHandle[] = []; + for (const value of Object.values(params)) { + if ( + value instanceof Object && + 'uid' in value && + typeof value.uid === 'string' && + Object.keys(value).length === 1 + ) { + handles.push(await request.page.getElementByUid(value.uid)); + } + } + const toolGroup = request.page.getInPageTools(); const tool = toolGroup?.tools.find(t => t.name === toolName); if (!tool) { @@ -102,7 +123,19 @@ export const executeInPageTool = definePageTool({ } const result = await request.page.pptrPage.evaluate( - async (name, args) => { + async (name, args, ...elements) => { + // Replace the UIDs with DOM elements. + for (const [key, value] of Object.entries(args)) { + if ( + value instanceof Object && + 'uid' in value && + typeof value.uid === 'string' && + Object.keys(value).length === 1 + ) { + args[key] = elements.shift(); + } + } + if (!window.__dtmcp?.executeTool) { throw new Error('No tools found on the page'); } @@ -114,6 +147,7 @@ export const executeInPageTool = definePageTool({ }, toolName, params, + ...handles, ); response.appendResponseLine(JSON.stringify(result, null, 2)); }, diff --git a/tests/McpResponse.test.ts b/tests/McpResponse.test.ts index 66be61e84..915377197 100644 --- a/tests/McpResponse.test.ts +++ b/tests/McpResponse.test.ts @@ -15,6 +15,8 @@ import sinon from 'sinon'; import type {ParsedArguments} from '../src/bin/chrome-devtools-mcp-cli-options.js'; import type {McpContext} from '../src/McpContext.js'; import type {McpResponse} from '../src/McpResponse.js'; +import {replaceHtmlElementsWithUids} from '../src/McpResponse.js'; +import type {JSONSchema7Definition} from '../src/third_party/index.js'; import { closePage, listPages, @@ -1205,3 +1207,251 @@ describe('inPage tools', () => { }, 'new_page'); }); }); + +describe('replaceHtmlElementsWithUids', () => { + it('does nothing for boolean schemas', () => { + const schemaTrue: JSONSchema7Definition = true; + const schemaFalse: JSONSchema7Definition = false; + + replaceHtmlElementsWithUids(schemaTrue); + replaceHtmlElementsWithUids(schemaFalse); + + assert.strictEqual(schemaTrue, true); + assert.strictEqual(schemaFalse, false); + }); + + it('replaces HTMLElement type with uid string', () => { + const schema: JSONSchema7Definition = { + type: 'object', + properties: { + foo: {type: 'string'}, + bar: {type: 'number'}, + }, + required: ['foo'], + }; + Object.assign(schema, {'x-mcp-type': 'HTMLElement'}); + + replaceHtmlElementsWithUids(schema); + + if (typeof schema === 'object') { + assert.deepStrictEqual(schema.properties, { + uid: {type: 'string'}, + }); + assert.deepStrictEqual(schema.required, ['uid']); + } else { + assert.fail('Schema should be an object'); + } + }); + + it('does not replace if x-mcp-type is not HTMLElement', () => { + const schema: JSONSchema7Definition = { + type: 'object', + properties: { + foo: {type: 'string'}, + }, + }; + Object.assign(schema, {'x-mcp-type': 'OtherType'}); + + replaceHtmlElementsWithUids(schema); + + if (typeof schema === 'object') { + assert.deepStrictEqual(schema.properties, { + foo: {type: 'string'}, + }); + assert.strictEqual(schema.required, undefined); + } else { + assert.fail('Schema should be an object'); + } + }); + + it('recurses into nested properties', () => { + const schema: JSONSchema7Definition = { + type: 'object', + properties: { + element: { + type: 'object', + properties: { + foo: {type: 'string'}, + }, + }, + other: { + type: 'string', + }, + }, + }; + if (typeof schema === 'object' && schema.properties) { + Object.assign(schema.properties.element, {'x-mcp-type': 'HTMLElement'}); + } + + replaceHtmlElementsWithUids(schema); + + if ( + typeof schema === 'object' && + schema.properties && + typeof schema.properties.element === 'object' + ) { + const elementSchema = schema.properties.element; + assert.deepStrictEqual(elementSchema.properties, { + uid: {type: 'string'}, + }); + assert.deepStrictEqual(elementSchema.required, ['uid']); + } else { + assert.fail('Unexpected schema structure'); + } + }); + + it('recurses into array items (single schema object)', () => { + const schema: JSONSchema7Definition = { + type: 'array', + items: { + type: 'object', + }, + }; + if (typeof schema === 'object' && typeof schema.items === 'object') { + Object.assign(schema.items, {'x-mcp-type': 'HTMLElement'}); + } + + replaceHtmlElementsWithUids(schema); + + if (typeof schema === 'object' && typeof schema.items === 'object') { + const itemsSchema = schema.items; + if (!Array.isArray(itemsSchema)) { + assert.deepStrictEqual(itemsSchema.properties, { + uid: {type: 'string'}, + }); + assert.deepStrictEqual(itemsSchema.required, ['uid']); + } else { + assert.fail('items should not be an array in this test case'); + } + } else { + assert.fail('Unexpected schema structure'); + } + }); + + it('recurses into array items (array of schemas)', () => { + const schema: JSONSchema7Definition = { + type: 'array', + items: [ + { + type: 'object', + }, + { + type: 'string', + }, + ], + }; + if (typeof schema === 'object' && Array.isArray(schema.items)) { + Object.assign(schema.items[0], {'x-mcp-type': 'HTMLElement'}); + } + + replaceHtmlElementsWithUids(schema); + + if (typeof schema === 'object' && Array.isArray(schema.items)) { + const firstItem = schema.items[0]; + if (typeof firstItem === 'object') { + assert.deepStrictEqual(firstItem.properties, { + uid: {type: 'string'}, + }); + assert.deepStrictEqual(firstItem.required, ['uid']); + } else { + assert.fail('First item should be an object'); + } + + const secondItem = schema.items[1]; + if (typeof secondItem === 'object') { + assert.strictEqual(secondItem.properties, undefined); + } else { + assert.fail('Second item should be an object'); + } + } else { + assert.fail('Unexpected schema structure'); + } + }); + + it('recurses into anyOf', () => { + const schema: JSONSchema7Definition = { + anyOf: [ + { + type: 'object', + }, + { + type: 'string', + }, + ], + }; + if (typeof schema === 'object' && Array.isArray(schema.anyOf)) { + Object.assign(schema.anyOf[0], {'x-mcp-type': 'HTMLElement'}); + } + + replaceHtmlElementsWithUids(schema); + + if (typeof schema === 'object' && Array.isArray(schema.anyOf)) { + const firstItem = schema.anyOf[0]; + if (typeof firstItem === 'object') { + assert.deepStrictEqual(firstItem.properties, { + uid: {type: 'string'}, + }); + } else { + assert.fail('First item should be an object'); + } + } else { + assert.fail('Unexpected schema structure'); + } + }); + + it('recurses into allOf', () => { + const schema: JSONSchema7Definition = { + allOf: [ + { + type: 'object', + }, + ], + }; + if (typeof schema === 'object' && Array.isArray(schema.allOf)) { + Object.assign(schema.allOf[0], {'x-mcp-type': 'HTMLElement'}); + } + + replaceHtmlElementsWithUids(schema); + + if (typeof schema === 'object' && Array.isArray(schema.allOf)) { + const firstItem = schema.allOf[0]; + if (typeof firstItem === 'object') { + assert.deepStrictEqual(firstItem.properties, { + uid: {type: 'string'}, + }); + } else { + assert.fail('First item should be an object'); + } + } else { + assert.fail('Unexpected schema structure'); + } + }); + + it('recurses into oneOf', () => { + const schema: JSONSchema7Definition = { + oneOf: [ + { + type: 'object', + }, + ], + }; + if (typeof schema === 'object' && Array.isArray(schema.oneOf)) { + Object.assign(schema.oneOf[0], {'x-mcp-type': 'HTMLElement'}); + } + + replaceHtmlElementsWithUids(schema); + + if (typeof schema === 'object' && Array.isArray(schema.oneOf)) { + const firstItem = schema.oneOf[0]; + if (typeof firstItem === 'object') { + assert.deepStrictEqual(firstItem.properties, { + uid: {type: 'string'}, + }); + } else { + assert.fail('First item should be an object'); + } + } else { + assert.fail('Unexpected schema structure'); + } + }); +}); diff --git a/tests/tools/inPage.test.ts b/tests/tools/inPage.test.ts index 622da9034..d87bbea83 100644 --- a/tests/tools/inPage.test.ts +++ b/tests/tools/inPage.test.ts @@ -347,5 +347,98 @@ describe('inPage', () => { {categoryInPageTools: true} as ParsedArguments, ); }); + + it('replaces uid with element handle in params', async () => { + await withMcpContext(async (response, context) => { + const page = await context.newPage(); + response.setPage(page); + + page.inPageTools = { + name: 'test-group', + description: 'test description', + tools: [ + { + name: 'test-tool', + description: 'test tool description', + inputSchema: { + type: 'object', + properties: { + element: {type: 'object'}, + }, + required: ['element'], + }, + }, + ], + }; + + await page.pptrPage.evaluate(() => { + window.__dtmcp = { + executeTool: async ( + _name: string, + args: Record, + ) => { + const el = args.element; + if (el instanceof HTMLElement) { + return { + isElement: true, + tagName: el.tagName, + id: el.id, + }; + } + return { + isElement: false, + tagName: '', + id: '', + }; + }, + }; + }); + + await page.pptrPage.evaluate(() => { + const div = document.createElement('div'); + div.id = 'test-id'; + document.body.appendChild(div); + }); + + const handle = await page.pptrPage.$('#test-id'); + if (!handle) { + throw new Error('Handle not found'); + } + + page.getElementByUid = async (uid: string) => { + if (uid === 'some-uid') { + return handle; + } + throw new Error('Not found'); + }; + + await executeInPageTool.handler( + { + params: { + toolName: 'test-tool', + params: JSON.stringify({element: {uid: 'some-uid'}}), + }, + page: page, + }, + response, + context, + ); + + assert.strictEqual( + response.responseLines[0], + JSON.stringify( + { + result: { + isElement: true, + tagName: 'DIV', + id: 'test-id', + }, + }, + null, + 2, + ), + ); + }); + }); }); });