Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion package.json
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@
"typecheck": "tsc --noEmit",
"format": "eslint --cache --fix . && prettier --write --cache .",
"check-format": "eslint --cache . && prettier --check --cache .;",
"gen": "npm run build && npm run docs:generate && npm run cli:generate && npm run format",
"gen": "npm run build && npm run docs:generate && npm run cli:generate && npm run update-tool-call-metrics && npm run format",
"docs:generate": "node --experimental-strip-types scripts/generate-docs.ts",
"start": "npm run build && node build/src/index.js",
"start-debug": "DEBUG=mcp:* DEBUG_COLORS=false npm run build && node build/src/index.js",
Expand All @@ -27,6 +27,7 @@
"prepare": "node --experimental-strip-types scripts/prepare.ts",
"verify-server-json-version": "node --experimental-strip-types scripts/verify-server-json-version.ts",
"update-lighthouse": "node --experimental-strip-types scripts/update-lighthouse.ts",
"update-tool-call-metrics": "node --experimental-strip-types scripts/update_tool_call_metrics.ts",
Comment thread
yulunz marked this conversation as resolved.
"verify-npm-package": "node scripts/verify-npm-package.mjs",
"eval": "npm run build && node --experimental-strip-types scripts/eval_gemini.ts",
"count-tokens": "node --experimental-strip-types scripts/count_tokens.ts"
Expand Down
51 changes: 51 additions & 0 deletions scripts/update_tool_call_metrics.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,51 @@
/**
* @license
* Copyright 2026 Google LLC
* SPDX-License-Identifier: Apache-2.0
*/

import * as fs from 'node:fs';
import * as path from 'node:path';

import type {ParsedArguments} from '../build/src/bin/chrome-devtools-mcp-cli-options.js';
import {generateToolMetrics} from '../build/src/telemetry/toolMetricsUtils.js';
import type {ToolDefinition} from '../build/src/tools/ToolDefinition.js';
import {createTools} from '../build/src/tools/tools.js';

export function HaveUniqueNames(tools: ToolDefinition[]): boolean {
const toolNames = tools.map(tool => tool.name);
const toolNamesSet = new Set(toolNames);
return toolNamesSet.size === toolNames.length;
}

function writeToolCallMetricsConfig() {
const outputPath = path.resolve('src/telemetry/tool_call_metrics.json');

const dir = path.dirname(outputPath);
if (!fs.existsSync(dir)) {
throw new Error(`Error: Directory ${dir} does not exist.`);
}

const fullTools = createTools({slim: false} as ParsedArguments);
const slimTools = createTools({slim: true} as ParsedArguments);

const allTools = [...fullTools, ...slimTools];

if (!HaveUniqueNames(allTools)) {
throw new Error('Error: Duplicate tool names found.');
}

// Map tools to their metadata
const toolData = generateToolMetrics(allTools);

// Sort by name for determinism
toolData.sort((a, b) => a.name.localeCompare(b.name));

fs.writeFileSync(outputPath, JSON.stringify(toolData, null, 2) + '\n');

console.log(
`Successfully wrote ${toolData.length} tool names with arguments to ${outputPath}`,
);
}

writeToolCallMetricsConfig();
24 changes: 20 additions & 4 deletions src/telemetry/ClearcutLogger.ts
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ import {
import {WatchdogClient} from './WatchdogClient.js';

const MS_PER_DAY = 24 * 60 * 60 * 1000;
const PARAM_BLOCKLIST = new Set(['uid']);
export const PARAM_BLOCKLIST = new Set(['uid', 'reqid', 'msgid']);

const SUPPORTED_ZOD_TYPES = [
'ZodString',
Expand All @@ -36,7 +36,7 @@ function isZodType(type: string): type is ZodType {
return SUPPORTED_ZOD_TYPES.includes(type as ZodType);
}

function getZodType(zodType: zod.ZodTypeAny): ZodType {
export function getZodType(zodType: zod.ZodTypeAny): ZodType {
const def = zodType._def;
const typeName = def.typeName;

Expand All @@ -59,7 +59,7 @@ function getZodType(zodType: zod.ZodTypeAny): ZodType {

type LoggedToolCallArgValue = string | number | boolean;

function transformName(zodType: ZodType, name: string): string {
export function transformArgName(zodType: ZodType, name: string): string {
if (zodType === 'ZodString') {
return `${name}_length`;
} else if (zodType === 'ZodArray') {
Expand All @@ -69,6 +69,22 @@ function transformName(zodType: ZodType, name: string): string {
}
}

export function transformArgType(zodType: ZodType): string {
if (zodType === 'ZodString' || zodType === 'ZodArray') {
return 'number';
}
switch (zodType) {
case 'ZodNumber':
return 'number';
case 'ZodBoolean':
return 'boolean';
case 'ZodEnum':
return 'enum';
default:
throw new Error(`Unsupported zod type for tool parameter: ${zodType}`);
}
}

function transformValue(
zodType: ZodType,
value: unknown,
Expand Down Expand Up @@ -117,7 +133,7 @@ export function sanitizeParams(
`parameter ${name} has type ${zodType} but value ${value} is not of equivalent type`,
);
}
const transformedName = transformName(zodType, name);
const transformedName = transformArgName(zodType, name);
const transformedValue = transformValue(zodType, value);
transformed[transformedName] = transformedValue;
}
Expand Down
70 changes: 70 additions & 0 deletions src/telemetry/toolMetricsUtils.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,70 @@
/**
* @license
* Copyright 2026 Google LLC
* SPDX-License-Identifier: Apache-2.0
*/

import type {ToolDefinition} from '../tools/ToolDefinition.js';

import {
transformArgName,
transformArgType,
getZodType,
PARAM_BLOCKLIST,
} from './ClearcutLogger.js';

/**
* Validates that all values in an enum are of the homogeneous primitive type.
* Returns the primitive type string. Throws an error if heterogeneous.
*/
export function validateEnumHomogeneity(values: unknown[]): string {
const firstType = typeof values[0];
for (const val of values) {
if (typeof val !== firstType) {
throw new Error('Heterogeneous enum types found');
}
}
return firstType;
}

export interface ArgMetric {
name: string;
argType: string;
}

export interface ToolMetric {
name: string;
args: ArgMetric[];
}

/**
* Generates tool metrics from tool definitions.
*/
export function generateToolMetrics(tools: ToolDefinition[]): ToolMetric[] {
return tools.map(tool => {
const args: ArgMetric[] = [];

for (const [name, schema] of Object.entries(tool.schema)) {
if (PARAM_BLOCKLIST.has(name)) {
continue;
}
const zodType = getZodType(schema);
const transformedName = transformArgName(zodType, name);
let argType = transformArgType(zodType);

if (zodType === 'ZodEnum' && schema._def.values?.length > 0) {
argType = validateEnumHomogeneity(schema._def.values);
}

args.push({
name: transformedName,
argType,
});
}

return {
name: tool.name,
args,
};
});
}
Loading
Loading