diff --git a/scripts/update_tool_call_metrics.ts b/scripts/update_tool_call_metrics.ts index bbf017f4c..f28e2688d 100644 --- a/scripts/update_tool_call_metrics.ts +++ b/scripts/update_tool_call_metrics.ts @@ -8,7 +8,11 @@ import * as fs from 'node:fs'; import * as path from 'node:path'; import type {ParsedArguments} from '../build/src/bin/chrome-devtools-mcp-cli-options.js'; -import {generateToolMetrics} from '../build/src/telemetry/toolMetricsUtils.js'; +import { + applyToExistingMetrics, + generateToolMetrics, + type ToolMetric, +} from '../build/src/telemetry/toolMetricsUtils.js'; import type {ToolDefinition} from '../build/src/tools/ToolDefinition.js'; import {createTools} from '../build/src/tools/tools.js'; @@ -35,16 +39,26 @@ function writeToolCallMetricsConfig() { throw new Error('Error: Duplicate tool names found.'); } - // Map tools to their metadata - const toolData = generateToolMetrics(allTools); + let existingMetrics: ToolMetric[] = []; + if (fs.existsSync(outputPath)) { + try { + existingMetrics = JSON.parse( + fs.readFileSync(outputPath, 'utf8'), + ) as ToolMetric[]; + } catch { + console.warn( + `Warning: Failed to parse existing metrics from ${outputPath}. Starting fresh.`, + ); + } + } - // Sort by name for determinism - toolData.sort((a, b) => a.name.localeCompare(b.name)); + const newMetrics = generateToolMetrics(allTools); + const mergedMetrics = applyToExistingMetrics(existingMetrics, newMetrics); - fs.writeFileSync(outputPath, JSON.stringify(toolData, null, 2) + '\n'); + fs.writeFileSync(outputPath, JSON.stringify(mergedMetrics, null, 2) + '\n'); console.log( - `Successfully wrote ${toolData.length} tool names with arguments to ${outputPath}`, + `Successfully wrote ${mergedMetrics.length} total tool metrics (including deprecated ones) to ${outputPath}`, ); } diff --git a/src/telemetry/toolMetricsUtils.ts b/src/telemetry/toolMetricsUtils.ts index 124913a68..30ba81b53 100644 --- a/src/telemetry/toolMetricsUtils.ts +++ b/src/telemetry/toolMetricsUtils.ts @@ -30,11 +30,61 @@ export function validateEnumHomogeneity(values: unknown[]): string { export interface ArgMetric { name: string; argType: string; + isDeprecated?: boolean; } export interface ToolMetric { name: string; args: ArgMetric[]; + isDeprecated?: boolean; +} + +export function applyToExistingMetrics( + existing: ToolMetric[], + update: ToolMetric[], +): ToolMetric[] { + const updated = applyToExisting(existing, update); + const existingByName = new Map(existing.map(tool => [tool.name, tool])); + const updatedByName = new Map(update.map(tool => [tool.name, tool])); + + return updated.map(tool => { + const existingTool = existingByName.get(tool.name); + const updatedTool = updatedByName.get(tool.name); + // If the tool still exists in the update, we will update the args. + if (existingTool && updatedTool) { + const updatedArgs = applyToExisting( + existingTool.args, + updatedTool.args, + ); + return {...tool, args: updatedArgs}; + } + return tool; + }); +} + +function applyToExisting( + existing: T[], + update: T[], +): T[] { + const existingNames = new Set(existing.map(item => item.name)); + const updatedNames = new Set(update.map(item => item.name)); + + const result: T[] = []; + // Keep the original ordering. + for (const entry of existing) { + const toAdd = {...entry}; + if (!updatedNames.has(entry.name)) { + toAdd.isDeprecated = true; + } + result.push(toAdd); + } + // New entries must be added to the very back of the list. + for (const entry of update) { + if (!existingNames.has(entry.name)) { + result.push({...entry}); + } + } + return result; } /** diff --git a/tests/telemetry/toolMetricsUtils.test.ts b/tests/telemetry/toolMetricsUtils.test.ts index a984c4287..0c369aaea 100644 --- a/tests/telemetry/toolMetricsUtils.test.ts +++ b/tests/telemetry/toolMetricsUtils.test.ts @@ -8,6 +8,7 @@ import assert from 'node:assert'; import {describe, it} from 'node:test'; import { + applyToExistingMetrics, generateToolMetrics, validateEnumHomogeneity, } from '../../src/telemetry/toolMetricsUtils.js'; @@ -80,4 +81,183 @@ describe('toolMetricsUtils', () => { assert.strictEqual(metrics[0].args[0].argType, 'string'); }); }); + + describe('applyToExistingMetrics', () => { + it('should return the same metrics if existing and update are the same', () => { + const existing = [{name: 'foo', args: []}]; + const update = [{name: 'foo', args: []}]; + const result = applyToExistingMetrics(existing, update); + const expected = [{name: 'foo', args: []}]; + assert.deepStrictEqual(result, expected); + }); + + it('should append new entries to the end of the array', () => { + const existing = [{name: 'foo', args: []}]; + const update = [ + {name: 'foo', args: []}, + {name: 'bar', args: []}, + ]; + const result = applyToExistingMetrics(existing, update); + const expected = [ + {name: 'foo', args: []}, + {name: 'bar', args: []}, + ]; + assert.deepStrictEqual(result, expected); + }); + + it('should mark missing entries as deprecated and preserve their order', () => { + const existing = [ + {name: 'foo', args: []}, + {name: 'bar', args: []}, + ]; + const update = [{name: 'foo', args: []}]; + const result = applyToExistingMetrics(existing, update); + const expected = [ + {name: 'foo', args: []}, + {name: 'bar', args: [], isDeprecated: true}, + ]; + assert.deepStrictEqual(result, expected); + }); + + it('should handle adding new entries and deprecating old ones simultaneously', () => { + const existing = [ + {name: 'foo', args: []}, + {name: 'bar', args: []}, + ]; + const update = [ + {name: 'bar', args: []}, + {name: 'baz', args: []}, + ]; + const result = applyToExistingMetrics(existing, update); + const expected = [ + {name: 'foo', args: [], isDeprecated: true}, + {name: 'bar', args: []}, + {name: 'baz', args: []}, + ]; + assert.deepStrictEqual(result, expected); + }); + + it('should append new arguments to the back', () => { + const existing = [ + {name: 'foo', args: [{name: 'arg_a', argType: 'string'}]}, + ]; + const update = [ + { + name: 'foo', + args: [ + {name: 'arg_a', argType: 'string'}, + {name: 'arg_b', argType: 'string'}, + ], + }, + ]; + const result = applyToExistingMetrics(existing, update); + const expected = [ + { + name: 'foo', + args: [ + {name: 'arg_a', argType: 'string'}, + {name: 'arg_b', argType: 'string'}, + ], + }, + ]; + assert.deepStrictEqual(result, expected); + }); + + it('should mark removed arguments as deprecated', () => { + const existing = [ + { + name: 'foo', + args: [ + {name: 'arg_a', argType: 'string'}, + {name: 'arg_b', argType: 'string'}, + ], + }, + ]; + const update = [ + {name: 'foo', args: [{name: 'arg_a', argType: 'string'}]}, + ]; + const result = applyToExistingMetrics(existing, update); + const expected = [ + { + name: 'foo', + args: [ + {name: 'arg_a', argType: 'string'}, + {name: 'arg_b', argType: 'string', isDeprecated: true}, + ], + }, + ]; + assert.deepStrictEqual(result, expected); + }); + + it('should not change args if they are the same', () => { + const existing = [ + {name: 'foo', args: [{name: 'arg_a', argType: 'string'}]}, + ]; + const update = [ + {name: 'foo', args: [{name: 'arg_a', argType: 'string'}]}, + ]; + const result = applyToExistingMetrics(existing, update); + const expected = [ + {name: 'foo', args: [{name: 'arg_a', argType: 'string'}]}, + ]; + assert.deepStrictEqual(result, expected); + }); + + it('should handle adding and removing arguments simultaneously', () => { + const existing = [ + { + name: 'foo', + args: [ + {name: 'arg_a', argType: 'string'}, + {name: 'arg_b', argType: 'string'}, + ], + }, + ]; + const update = [ + { + name: 'foo', + args: [ + {name: 'arg_b', argType: 'string'}, + {name: 'arg_c', argType: 'string'}, + ], + }, + ]; + const result = applyToExistingMetrics(existing, update); + const expected = [ + { + name: 'foo', + args: [ + {name: 'arg_a', argType: 'string', isDeprecated: true}, + {name: 'arg_b', argType: 'string'}, + {name: 'arg_c', argType: 'string'}, + ], + }, + ]; + assert.deepStrictEqual(result, expected); + }); + + it('should handle tool and argument changes simultaneously', () => { + const existing = [ + {name: 'foo', args: [{name: 'arg_a', argType: 'string'}]}, + {name: 'bar', args: []}, + ]; + const update = [ + {name: 'foo', args: [{name: 'arg_b', argType: 'string'}]}, + {name: 'baz', args: []}, + ]; + const result = applyToExistingMetrics(existing, update); + const expected = [ + { + name: 'foo', + args: [ + {name: 'arg_a', argType: 'string', isDeprecated: true}, + {name: 'arg_b', argType: 'string'}, + ], + }, + {name: 'bar', args: [], isDeprecated: true}, + {name: 'baz', args: []}, + ]; + assert.deepStrictEqual(result, expected); + }); + }); });