From 20bbd5503f45f5cdad14f4dca08c61b7daf2ad6e Mon Sep 17 00:00:00 2001 From: Yulun Zeng <11618243+yulunz@users.noreply.github.com> Date: Thu, 26 Mar 2026 22:40:57 +0000 Subject: [PATCH] Add a script to generate tool_call_metrics.json for telemetry. --- package.json | 3 +- scripts/update_tool_call_metrics.ts | 51 +++ src/telemetry/ClearcutLogger.ts | 24 +- src/telemetry/toolMetricsUtils.ts | 70 +++ src/telemetry/tool_call_metrics.json | 543 +++++++++++++++++++++++ tests/telemetry/toolMetricsUtils.test.ts | 83 ++++ 6 files changed, 769 insertions(+), 5 deletions(-) create mode 100644 scripts/update_tool_call_metrics.ts create mode 100644 src/telemetry/toolMetricsUtils.ts create mode 100644 src/telemetry/tool_call_metrics.json create mode 100644 tests/telemetry/toolMetricsUtils.test.ts diff --git a/package.json b/package.json index 7093c0859..731b2fb6a 100644 --- a/package.json +++ b/package.json @@ -16,7 +16,7 @@ "typecheck": "tsc --noEmit", "format": "eslint --cache --fix . && prettier --write --cache .", "check-format": "eslint --cache . && prettier --check --cache .;", - "gen": "npm run build && npm run docs:generate && npm run cli:generate && npm run format", + "gen": "npm run build && npm run docs:generate && npm run cli:generate && npm run update-tool-call-metrics && npm run format", "docs:generate": "node --experimental-strip-types scripts/generate-docs.ts", "start": "npm run build && node build/src/index.js", "start-debug": "DEBUG=mcp:* DEBUG_COLORS=false npm run build && node build/src/index.js", @@ -27,6 +27,7 @@ "prepare": "node --experimental-strip-types scripts/prepare.ts", "verify-server-json-version": "node --experimental-strip-types scripts/verify-server-json-version.ts", "update-lighthouse": "node --experimental-strip-types scripts/update-lighthouse.ts", + "update-tool-call-metrics": "node --experimental-strip-types scripts/update_tool_call_metrics.ts", "verify-npm-package": "node scripts/verify-npm-package.mjs", "eval": "npm run build && node --experimental-strip-types scripts/eval_gemini.ts", "count-tokens": "node --experimental-strip-types scripts/count_tokens.ts" diff --git a/scripts/update_tool_call_metrics.ts b/scripts/update_tool_call_metrics.ts new file mode 100644 index 000000000..bbf017f4c --- /dev/null +++ b/scripts/update_tool_call_metrics.ts @@ -0,0 +1,51 @@ +/** + * @license + * Copyright 2026 Google LLC + * SPDX-License-Identifier: Apache-2.0 + */ + +import * as fs from 'node:fs'; +import * as path from 'node:path'; + +import type {ParsedArguments} from '../build/src/bin/chrome-devtools-mcp-cli-options.js'; +import {generateToolMetrics} from '../build/src/telemetry/toolMetricsUtils.js'; +import type {ToolDefinition} from '../build/src/tools/ToolDefinition.js'; +import {createTools} from '../build/src/tools/tools.js'; + +export function HaveUniqueNames(tools: ToolDefinition[]): boolean { + const toolNames = tools.map(tool => tool.name); + const toolNamesSet = new Set(toolNames); + return toolNamesSet.size === toolNames.length; +} + +function writeToolCallMetricsConfig() { + const outputPath = path.resolve('src/telemetry/tool_call_metrics.json'); + + const dir = path.dirname(outputPath); + if (!fs.existsSync(dir)) { + throw new Error(`Error: Directory ${dir} does not exist.`); + } + + const fullTools = createTools({slim: false} as ParsedArguments); + const slimTools = createTools({slim: true} as ParsedArguments); + + const allTools = [...fullTools, ...slimTools]; + + if (!HaveUniqueNames(allTools)) { + throw new Error('Error: Duplicate tool names found.'); + } + + // Map tools to their metadata + const toolData = generateToolMetrics(allTools); + + // Sort by name for determinism + toolData.sort((a, b) => a.name.localeCompare(b.name)); + + fs.writeFileSync(outputPath, JSON.stringify(toolData, null, 2) + '\n'); + + console.log( + `Successfully wrote ${toolData.length} tool names with arguments to ${outputPath}`, + ); +} + +writeToolCallMetricsConfig(); diff --git a/src/telemetry/ClearcutLogger.ts b/src/telemetry/ClearcutLogger.ts index a53345746..82f766cd4 100644 --- a/src/telemetry/ClearcutLogger.ts +++ b/src/telemetry/ClearcutLogger.ts @@ -21,7 +21,7 @@ import { import {WatchdogClient} from './WatchdogClient.js'; const MS_PER_DAY = 24 * 60 * 60 * 1000; -const PARAM_BLOCKLIST = new Set(['uid']); +export const PARAM_BLOCKLIST = new Set(['uid', 'reqid', 'msgid']); const SUPPORTED_ZOD_TYPES = [ 'ZodString', @@ -36,7 +36,7 @@ function isZodType(type: string): type is ZodType { return SUPPORTED_ZOD_TYPES.includes(type as ZodType); } -function getZodType(zodType: zod.ZodTypeAny): ZodType { +export function getZodType(zodType: zod.ZodTypeAny): ZodType { const def = zodType._def; const typeName = def.typeName; @@ -59,7 +59,7 @@ function getZodType(zodType: zod.ZodTypeAny): ZodType { type LoggedToolCallArgValue = string | number | boolean; -function transformName(zodType: ZodType, name: string): string { +export function transformArgName(zodType: ZodType, name: string): string { if (zodType === 'ZodString') { return `${name}_length`; } else if (zodType === 'ZodArray') { @@ -69,6 +69,22 @@ function transformName(zodType: ZodType, name: string): string { } } +export function transformArgType(zodType: ZodType): string { + if (zodType === 'ZodString' || zodType === 'ZodArray') { + return 'number'; + } + switch (zodType) { + case 'ZodNumber': + return 'number'; + case 'ZodBoolean': + return 'boolean'; + case 'ZodEnum': + return 'enum'; + default: + throw new Error(`Unsupported zod type for tool parameter: ${zodType}`); + } +} + function transformValue( zodType: ZodType, value: unknown, @@ -117,7 +133,7 @@ export function sanitizeParams( `parameter ${name} has type ${zodType} but value ${value} is not of equivalent type`, ); } - const transformedName = transformName(zodType, name); + const transformedName = transformArgName(zodType, name); const transformedValue = transformValue(zodType, value); transformed[transformedName] = transformedValue; } diff --git a/src/telemetry/toolMetricsUtils.ts b/src/telemetry/toolMetricsUtils.ts new file mode 100644 index 000000000..124913a68 --- /dev/null +++ b/src/telemetry/toolMetricsUtils.ts @@ -0,0 +1,70 @@ +/** + * @license + * Copyright 2026 Google LLC + * SPDX-License-Identifier: Apache-2.0 + */ + +import type {ToolDefinition} from '../tools/ToolDefinition.js'; + +import { + transformArgName, + transformArgType, + getZodType, + PARAM_BLOCKLIST, +} from './ClearcutLogger.js'; + +/** + * Validates that all values in an enum are of the homogeneous primitive type. + * Returns the primitive type string. Throws an error if heterogeneous. + */ +export function validateEnumHomogeneity(values: unknown[]): string { + const firstType = typeof values[0]; + for (const val of values) { + if (typeof val !== firstType) { + throw new Error('Heterogeneous enum types found'); + } + } + return firstType; +} + +export interface ArgMetric { + name: string; + argType: string; +} + +export interface ToolMetric { + name: string; + args: ArgMetric[]; +} + +/** + * Generates tool metrics from tool definitions. + */ +export function generateToolMetrics(tools: ToolDefinition[]): ToolMetric[] { + return tools.map(tool => { + const args: ArgMetric[] = []; + + for (const [name, schema] of Object.entries(tool.schema)) { + if (PARAM_BLOCKLIST.has(name)) { + continue; + } + const zodType = getZodType(schema); + const transformedName = transformArgName(zodType, name); + let argType = transformArgType(zodType); + + if (zodType === 'ZodEnum' && schema._def.values?.length > 0) { + argType = validateEnumHomogeneity(schema._def.values); + } + + args.push({ + name: transformedName, + argType, + }); + } + + return { + name: tool.name, + args, + }; + }); +} diff --git a/src/telemetry/tool_call_metrics.json b/src/telemetry/tool_call_metrics.json new file mode 100644 index 000000000..b5437fb9d --- /dev/null +++ b/src/telemetry/tool_call_metrics.json @@ -0,0 +1,543 @@ +[ + { + "name": "click", + "args": [ + { + "name": "dblClick", + "argType": "boolean" + }, + { + "name": "includeSnapshot", + "argType": "boolean" + } + ] + }, + { + "name": "click_at", + "args": [ + { + "name": "x", + "argType": "number" + }, + { + "name": "y", + "argType": "number" + }, + { + "name": "dblClick", + "argType": "boolean" + }, + { + "name": "includeSnapshot", + "argType": "boolean" + } + ] + }, + { + "name": "close_page", + "args": [ + { + "name": "pageId", + "argType": "number" + } + ] + }, + { + "name": "drag", + "args": [ + { + "name": "from_uid_length", + "argType": "number" + }, + { + "name": "to_uid_length", + "argType": "number" + }, + { + "name": "includeSnapshot", + "argType": "boolean" + } + ] + }, + { + "name": "emulate", + "args": [ + { + "name": "networkConditions", + "argType": "enum" + }, + { + "name": "cpuThrottlingRate", + "argType": "number" + }, + { + "name": "geolocation_length", + "argType": "number" + }, + { + "name": "userAgent_length", + "argType": "number" + }, + { + "name": "colorScheme", + "argType": "enum" + }, + { + "name": "viewport_length", + "argType": "number" + } + ] + }, + { + "name": "evaluate", + "args": [ + { + "name": "script_length", + "argType": "number" + } + ] + }, + { + "name": "evaluate_script", + "args": [ + { + "name": "function_length", + "argType": "number" + }, + { + "name": "args_count", + "argType": "number" + } + ] + }, + { + "name": "execute_in_page_tool", + "args": [ + { + "name": "toolName_length", + "argType": "number" + }, + { + "name": "params_length", + "argType": "number" + } + ] + }, + { + "name": "fill", + "args": [ + { + "name": "value_length", + "argType": "number" + }, + { + "name": "includeSnapshot", + "argType": "boolean" + } + ] + }, + { + "name": "fill_form", + "args": [ + { + "name": "elements_count", + "argType": "number" + }, + { + "name": "includeSnapshot", + "argType": "boolean" + } + ] + }, + { + "name": "get_console_message", + "args": [] + }, + { + "name": "get_network_request", + "args": [ + { + "name": "requestFilePath_length", + "argType": "number" + }, + { + "name": "responseFilePath_length", + "argType": "number" + } + ] + }, + { + "name": "get_tab_id", + "args": [ + { + "name": "pageId", + "argType": "number" + } + ] + }, + { + "name": "handle_dialog", + "args": [ + { + "name": "action", + "argType": "string" + }, + { + "name": "promptText_length", + "argType": "number" + } + ] + }, + { + "name": "hover", + "args": [ + { + "name": "includeSnapshot", + "argType": "boolean" + } + ] + }, + { + "name": "install_extension", + "args": [ + { + "name": "path_length", + "argType": "number" + } + ] + }, + { + "name": "lighthouse_audit", + "args": [ + { + "name": "mode", + "argType": "enum" + }, + { + "name": "device", + "argType": "enum" + }, + { + "name": "outputDirPath_length", + "argType": "number" + } + ] + }, + { + "name": "list_console_messages", + "args": [ + { + "name": "pageSize", + "argType": "number" + }, + { + "name": "pageIdx", + "argType": "number" + }, + { + "name": "types_count", + "argType": "number" + }, + { + "name": "includePreservedMessages", + "argType": "boolean" + } + ] + }, + { + "name": "list_extensions", + "args": [] + }, + { + "name": "list_in_page_tools", + "args": [] + }, + { + "name": "list_network_requests", + "args": [ + { + "name": "pageSize", + "argType": "number" + }, + { + "name": "pageIdx", + "argType": "number" + }, + { + "name": "resourceTypes_count", + "argType": "number" + }, + { + "name": "includePreservedRequests", + "argType": "boolean" + } + ] + }, + { + "name": "list_pages", + "args": [] + }, + { + "name": "navigate", + "args": [ + { + "name": "url_length", + "argType": "number" + } + ] + }, + { + "name": "navigate_page", + "args": [ + { + "name": "type", + "argType": "enum" + }, + { + "name": "url_length", + "argType": "number" + }, + { + "name": "ignoreCache", + "argType": "boolean" + }, + { + "name": "handleBeforeUnload", + "argType": "enum" + }, + { + "name": "initScript_length", + "argType": "number" + }, + { + "name": "timeout", + "argType": "number" + } + ] + }, + { + "name": "new_page", + "args": [ + { + "name": "url_length", + "argType": "number" + }, + { + "name": "background", + "argType": "boolean" + }, + { + "name": "isolatedContext_length", + "argType": "number" + }, + { + "name": "timeout", + "argType": "number" + } + ] + }, + { + "name": "performance_analyze_insight", + "args": [ + { + "name": "insightSetId_length", + "argType": "number" + }, + { + "name": "insightName_length", + "argType": "number" + } + ] + }, + { + "name": "performance_start_trace", + "args": [ + { + "name": "reload", + "argType": "boolean" + }, + { + "name": "autoStop", + "argType": "boolean" + }, + { + "name": "filePath_length", + "argType": "number" + } + ] + }, + { + "name": "performance_stop_trace", + "args": [ + { + "name": "filePath_length", + "argType": "number" + } + ] + }, + { + "name": "press_key", + "args": [ + { + "name": "key_length", + "argType": "number" + }, + { + "name": "includeSnapshot", + "argType": "boolean" + } + ] + }, + { + "name": "reload_extension", + "args": [ + { + "name": "id_length", + "argType": "number" + } + ] + }, + { + "name": "resize_page", + "args": [ + { + "name": "width", + "argType": "number" + }, + { + "name": "height", + "argType": "number" + } + ] + }, + { + "name": "screencast_start", + "args": [ + { + "name": "path_length", + "argType": "number" + } + ] + }, + { + "name": "screencast_stop", + "args": [] + }, + { + "name": "screenshot", + "args": [] + }, + { + "name": "select_page", + "args": [ + { + "name": "pageId", + "argType": "number" + }, + { + "name": "bringToFront", + "argType": "boolean" + } + ] + }, + { + "name": "take_memory_snapshot", + "args": [ + { + "name": "filePath_length", + "argType": "number" + } + ] + }, + { + "name": "take_screenshot", + "args": [ + { + "name": "format", + "argType": "enum" + }, + { + "name": "quality", + "argType": "number" + }, + { + "name": "fullPage", + "argType": "boolean" + }, + { + "name": "filePath_length", + "argType": "number" + } + ] + }, + { + "name": "take_snapshot", + "args": [ + { + "name": "verbose", + "argType": "boolean" + }, + { + "name": "filePath_length", + "argType": "number" + } + ] + }, + { + "name": "trigger_extension_action", + "args": [ + { + "name": "id_length", + "argType": "number" + } + ] + }, + { + "name": "type_text", + "args": [ + { + "name": "text_length", + "argType": "number" + }, + { + "name": "submitKey_length", + "argType": "number" + } + ] + }, + { + "name": "uninstall_extension", + "args": [ + { + "name": "id_length", + "argType": "number" + } + ] + }, + { + "name": "upload_file", + "args": [ + { + "name": "filePath_length", + "argType": "number" + }, + { + "name": "includeSnapshot", + "argType": "boolean" + } + ] + }, + { + "name": "wait_for", + "args": [ + { + "name": "text_count", + "argType": "number" + }, + { + "name": "timeout", + "argType": "number" + } + ] + } +] diff --git a/tests/telemetry/toolMetricsUtils.test.ts b/tests/telemetry/toolMetricsUtils.test.ts new file mode 100644 index 000000000..a984c4287 --- /dev/null +++ b/tests/telemetry/toolMetricsUtils.test.ts @@ -0,0 +1,83 @@ +/** + * @license + * Copyright 2026 Google LLC + * SPDX-License-Identifier: Apache-2.0 + */ + +import assert from 'node:assert'; +import {describe, it} from 'node:test'; + +import { + generateToolMetrics, + validateEnumHomogeneity, +} from '../../src/telemetry/toolMetricsUtils.js'; +import {zod} from '../../src/third_party/index.js'; +import {ToolCategory} from '../../src/tools/categories.js'; +import type {ToolDefinition} from '../../src/tools/ToolDefinition.js'; + +describe('toolMetricsUtils', () => { + describe('validateEnumHomogeneity', () => { + it('should return the primitive type of a homogeneous enum', () => { + const result = validateEnumHomogeneity(['a', 'b', 'c']); + assert.strictEqual(result, 'string'); + + const result2 = validateEnumHomogeneity([1, 2, 3]); + assert.strictEqual(result2, 'number'); + }); + + it('should throw for heterogeneous enum types', () => { + assert.throws(() => { + validateEnumHomogeneity(['a', 1, 'c']); + }, /Heterogeneous enum types found/); + }); + }); + + describe('generateToolMetrics', () => { + it('should map tools correctly and apply transformations', () => { + const mockTool: ToolDefinition = { + name: 'test_tool', + description: 'test description', + annotations: { + category: ToolCategory.INPUT, + readOnlyHint: true, + }, + schema: { + argStr: zod.string(), + uid: zod.string(), // Should be blocked + }, + handler: async () => { + // no-op + }, + }; + + const metrics = generateToolMetrics([mockTool]); + assert.strictEqual(metrics.length, 1); + assert.strictEqual(metrics[0].name, 'test_tool'); + assert.strictEqual(metrics[0].args.length, 1); // uid is blocked + assert.strictEqual(metrics[0].args[0].name, 'argStr_length'); + assert.strictEqual(metrics[0].args[0].argType, 'number'); + }); + + it('should handle enums correctly', () => { + const mockTool: ToolDefinition = { + name: 'enum_tool', + description: 'test description', + annotations: { + category: ToolCategory.INPUT, + readOnlyHint: true, + }, + schema: { + argEnum: zod.enum(['foo', 'bar']), + }, + handler: async () => { + // no-op + }, + }; + + const metrics = generateToolMetrics([mockTool]); + assert.strictEqual(metrics.length, 1); + assert.strictEqual(metrics[0].args[0].name, 'argEnum'); + assert.strictEqual(metrics[0].args[0].argType, 'string'); + }); + }); +});