Skip to content

Commit 6ad1ffe

Browse files
warren830tamas-la
warren830
authored and
tamas-la
committed
feat(q-dev): add Multi-AI Tool Comparison dashboard (Copilot vs Kiro) (#8794)
Add a Grafana dashboard comparing GitHub Copilot and Kiro side by side: - Weekly active users comparison - Code suggestions & acceptance events (per tool) - LOC accepted comparison (combined time series) - Acceptance rate comparison (bar gauge) Template variables for Copilot connection/scope selection. Data from _tool_copilot_enterprise_daily_metrics vs _tool_q_dev_user_report and _tool_q_dev_user_data.
1 parent 2592c2d commit 6ad1ffe

1 file changed

Lines changed: 299 additions & 0 deletions

File tree

‎grafana/dashboards/MultiAIComparison.json‎

Lines changed: 299 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,299 @@
1+
{
2+
"annotations": {
3+
"list": [
4+
{
5+
"builtIn": 1,
6+
"datasource": "-- Grafana --",
7+
"enable": true,
8+
"hide": true,
9+
"iconColor": "rgba(0, 211, 255, 1)",
10+
"name": "Annotations & Alerts",
11+
"type": "dashboard"
12+
}
13+
]
14+
},
15+
"editable": true,
16+
"fiscalYearStartMonth": 0,
17+
"graphTooltip": 1,
18+
"id": null,
19+
"links": [
20+
{
21+
"asDropdown": false,
22+
"icon": "external link",
23+
"includeVars": true,
24+
"keepTime": true,
25+
"tags": [],
26+
"targetBlank": true,
27+
"title": "Kiro Usage",
28+
"type": "link",
29+
"url": "/d/qdev_user_report"
30+
},
31+
{
32+
"asDropdown": false,
33+
"icon": "external link",
34+
"includeVars": true,
35+
"keepTime": true,
36+
"tags": [],
37+
"targetBlank": true,
38+
"title": "Copilot Adoption",
39+
"type": "link",
40+
"url": "/d/copilot_adoption"
41+
}
42+
],
43+
"panels": [
44+
{
45+
"datasource": {
46+
"type": "datasource",
47+
"uid": "grafana"
48+
},
49+
"gridPos": { "h": 3, "w": 24, "x": 0, "y": 0 },
50+
"id": 1,
51+
"options": {
52+
"code": { "language": "plaintext", "showLineNumbers": false, "showMiniMap": false },
53+
"content": "## Multi-AI Tool Comparison: Copilot vs Kiro\nCompare GitHub Copilot and Kiro (Amazon Q Developer) adoption and usage side by side.\n\n**Note:** Copilot metrics come from `_tool_copilot_enterprise_daily_metrics` (enterprise-level aggregates). Kiro metrics come from `_tool_q_dev_user_report` and `_tool_q_dev_user_data`. Select the Copilot connection and scope using the dropdowns above.",
54+
"mode": "markdown"
55+
},
56+
"title": "About",
57+
"type": "text"
58+
},
59+
{
60+
"collapsed": false,
61+
"gridPos": { "h": 1, "w": 24, "x": 0, "y": 3 },
62+
"id": 2,
63+
"panels": [],
64+
"title": "Active Users",
65+
"type": "row"
66+
},
67+
{
68+
"datasource": "mysql",
69+
"description": "Weekly active users for both tools",
70+
"fieldConfig": {
71+
"defaults": {
72+
"color": { "mode": "palette-classic" },
73+
"custom": {
74+
"axisBorderShow": false, "axisLabel": "", "axisPlacement": "auto",
75+
"drawStyle": "line", "fillOpacity": 10, "lineInterpolation": "smooth", "lineWidth": 2,
76+
"pointSize": 5, "showPoints": "never", "spanNulls": true,
77+
"stacking": { "mode": "none" }, "thresholdsStyle": { "mode": "off" }
78+
},
79+
"unit": "short"
80+
},
81+
"overrides": []
82+
},
83+
"gridPos": { "h": 8, "w": 24, "x": 0, "y": 4 },
84+
"id": 3,
85+
"options": {
86+
"legend": { "calcs": ["mean", "max"], "displayMode": "table", "placement": "right", "showLegend": true },
87+
"tooltip": { "mode": "multi" }
88+
},
89+
"targets": [
90+
{
91+
"datasource": "mysql",
92+
"format": "time_series",
93+
"rawQuery": true,
94+
"rawSql": "SELECT\n DATE_SUB(DATE(date), INTERVAL WEEKDAY(DATE(date)) DAY) AS time,\n COUNT(DISTINCT user_id) AS 'Kiro Active Users'\nFROM _tool_q_dev_user_report\nWHERE $__timeFilter(date)\nGROUP BY DATE_SUB(DATE(date), INTERVAL WEEKDAY(DATE(date)) DAY)\nORDER BY time",
95+
"refId": "A"
96+
},
97+
{
98+
"datasource": "mysql",
99+
"format": "time_series",
100+
"rawQuery": true,
101+
"rawSql": "SELECT\n DATE_SUB(DATE(day), INTERVAL WEEKDAY(DATE(day)) DAY) AS time,\n MAX(daily_active_users) AS 'Copilot Active Users'\nFROM _tool_copilot_enterprise_daily_metrics\nWHERE connection_id = ${copilot_connection_id}\n AND scope_id = '${copilot_scope_id}'\n AND $__timeFilter(day)\nGROUP BY DATE_SUB(DATE(day), INTERVAL WEEKDAY(DATE(day)) DAY)\nORDER BY time",
102+
"refId": "B"
103+
}
104+
],
105+
"title": "Weekly Active Users Comparison",
106+
"type": "timeseries"
107+
},
108+
{
109+
"collapsed": false,
110+
"gridPos": { "h": 1, "w": 24, "x": 0, "y": 12 },
111+
"id": 10,
112+
"panels": [],
113+
"title": "Code Generation",
114+
"type": "row"
115+
},
116+
{
117+
"datasource": "mysql",
118+
"description": "Weekly code generation and acceptance events",
119+
"fieldConfig": {
120+
"defaults": {
121+
"color": { "mode": "palette-classic" },
122+
"custom": {
123+
"axisBorderShow": false, "axisLabel": "", "axisPlacement": "auto",
124+
"drawStyle": "line", "fillOpacity": 10, "lineInterpolation": "smooth", "lineWidth": 2,
125+
"pointSize": 5, "showPoints": "never", "spanNulls": true,
126+
"stacking": { "mode": "none" }, "thresholdsStyle": { "mode": "off" }
127+
},
128+
"unit": "short"
129+
},
130+
"overrides": []
131+
},
132+
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 13 },
133+
"id": 11,
134+
"options": {
135+
"legend": { "calcs": ["mean", "sum"], "displayMode": "table", "placement": "right", "showLegend": true },
136+
"tooltip": { "mode": "multi" }
137+
},
138+
"targets": [
139+
{
140+
"datasource": "mysql",
141+
"format": "time_series",
142+
"rawQuery": true,
143+
"rawSql": "SELECT\n DATE_SUB(DATE(date), INTERVAL WEEKDAY(DATE(date)) DAY) AS time,\n SUM(inline_suggestions_count) AS 'Kiro: Suggestions',\n SUM(inline_acceptance_count) AS 'Kiro: Accepted'\nFROM _tool_q_dev_user_data\nWHERE $__timeFilter(date)\nGROUP BY DATE_SUB(DATE(date), INTERVAL WEEKDAY(DATE(date)) DAY)\nORDER BY time",
144+
"refId": "A"
145+
}
146+
],
147+
"title": "Kiro: Code Suggestions & Acceptance",
148+
"type": "timeseries"
149+
},
150+
{
151+
"datasource": "mysql",
152+
"description": "Weekly Copilot code generation and acceptance events",
153+
"fieldConfig": {
154+
"defaults": {
155+
"color": { "mode": "palette-classic" },
156+
"custom": {
157+
"axisBorderShow": false, "axisLabel": "", "axisPlacement": "auto",
158+
"drawStyle": "line", "fillOpacity": 10, "lineInterpolation": "smooth", "lineWidth": 2,
159+
"pointSize": 5, "showPoints": "never", "spanNulls": true,
160+
"stacking": { "mode": "none" }, "thresholdsStyle": { "mode": "off" }
161+
},
162+
"unit": "short"
163+
},
164+
"overrides": []
165+
},
166+
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 13 },
167+
"id": 12,
168+
"options": {
169+
"legend": { "calcs": ["mean", "sum"], "displayMode": "table", "placement": "right", "showLegend": true },
170+
"tooltip": { "mode": "multi" }
171+
},
172+
"targets": [
173+
{
174+
"datasource": "mysql",
175+
"format": "time_series",
176+
"rawQuery": true,
177+
"rawSql": "SELECT\n DATE_SUB(DATE(day), INTERVAL WEEKDAY(DATE(day)) DAY) AS time,\n SUM(code_generation_activity_count) AS 'Copilot: Suggestions',\n SUM(code_acceptance_activity_count) AS 'Copilot: Accepted'\nFROM _tool_copilot_enterprise_daily_metrics\nWHERE connection_id = ${copilot_connection_id}\n AND scope_id = '${copilot_scope_id}'\n AND $__timeFilter(day)\nGROUP BY DATE_SUB(DATE(day), INTERVAL WEEKDAY(DATE(day)) DAY)\nORDER BY time",
178+
"refId": "A"
179+
}
180+
],
181+
"title": "Copilot: Code Suggestions & Acceptance",
182+
"type": "timeseries"
183+
},
184+
{
185+
"collapsed": false,
186+
"gridPos": { "h": 1, "w": 24, "x": 0, "y": 21 },
187+
"id": 20,
188+
"panels": [],
189+
"title": "Lines of Code & Acceptance Rate",
190+
"type": "row"
191+
},
192+
{
193+
"datasource": "mysql",
194+
"description": "Weekly AI-generated lines of code accepted",
195+
"fieldConfig": {
196+
"defaults": {
197+
"color": { "mode": "palette-classic" },
198+
"custom": {
199+
"axisBorderShow": false, "axisLabel": "", "axisPlacement": "auto",
200+
"drawStyle": "line", "fillOpacity": 10, "lineInterpolation": "smooth", "lineWidth": 2,
201+
"pointSize": 5, "showPoints": "never", "spanNulls": true,
202+
"stacking": { "mode": "none" }, "thresholdsStyle": { "mode": "off" }
203+
},
204+
"unit": "short"
205+
},
206+
"overrides": []
207+
},
208+
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 22 },
209+
"id": 21,
210+
"options": {
211+
"legend": { "calcs": ["mean", "sum"], "displayMode": "table", "placement": "right", "showLegend": true },
212+
"tooltip": { "mode": "multi" }
213+
},
214+
"targets": [
215+
{
216+
"datasource": "mysql",
217+
"format": "time_series",
218+
"rawQuery": true,
219+
"rawSql": "SELECT time, SUM(kiro_loc) AS 'Kiro LOC Accepted', SUM(copilot_loc) AS 'Copilot LOC Added'\nFROM (\n SELECT DATE_SUB(DATE(date), INTERVAL WEEKDAY(DATE(date)) DAY) AS time,\n SUM(inline_ai_code_lines + chat_ai_code_lines) AS kiro_loc, 0 AS copilot_loc\n FROM _tool_q_dev_user_data WHERE $__timeFilter(date)\n GROUP BY DATE_SUB(DATE(date), INTERVAL WEEKDAY(DATE(date)) DAY)\n UNION ALL\n SELECT DATE_SUB(DATE(day), INTERVAL WEEKDAY(DATE(day)) DAY) AS time,\n 0 AS kiro_loc, SUM(loc_added_sum) AS copilot_loc\n FROM _tool_copilot_enterprise_daily_metrics\n WHERE connection_id = ${copilot_connection_id}\n AND scope_id = '${copilot_scope_id}'\n AND $__timeFilter(day)\n GROUP BY DATE_SUB(DATE(day), INTERVAL WEEKDAY(DATE(day)) DAY)\n) combined\nGROUP BY time ORDER BY time",
220+
"refId": "A"
221+
}
222+
],
223+
"title": "LOC Accepted: Kiro vs Copilot",
224+
"type": "timeseries"
225+
},
226+
{
227+
"datasource": "mysql",
228+
"description": "Overall acceptance rates for both tools",
229+
"fieldConfig": {
230+
"defaults": {
231+
"color": { "mode": "palette-classic" },
232+
"mappings": [],
233+
"thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] },
234+
"unit": "percent"
235+
},
236+
"overrides": []
237+
},
238+
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 22 },
239+
"id": 22,
240+
"options": {
241+
"barRadius": 0.1,
242+
"barWidth": 0.5,
243+
"orientation": "horizontal",
244+
"showValue": "auto",
245+
"stacking": "none",
246+
"tooltip": { "mode": "single" }
247+
},
248+
"targets": [
249+
{
250+
"datasource": "mysql",
251+
"format": "table",
252+
"rawQuery": true,
253+
"rawSql": "SELECT 'Kiro' AS Tool,\n ROUND(SUM(inline_acceptance_count) * 100.0 / NULLIF(SUM(inline_suggestions_count), 0), 1) AS 'Acceptance Rate'\nFROM _tool_q_dev_user_data\nWHERE $__timeFilter(date)\nUNION ALL\nSELECT 'Copilot' AS Tool,\n ROUND(SUM(code_acceptance_activity_count) * 100.0 / NULLIF(SUM(code_generation_activity_count), 0), 1)\nFROM _tool_copilot_enterprise_daily_metrics\nWHERE connection_id = ${copilot_connection_id}\n AND scope_id = '${copilot_scope_id}'\n AND $__timeFilter(day)",
254+
"refId": "A"
255+
}
256+
],
257+
"title": "Acceptance Rate Comparison",
258+
"type": "bargauge"
259+
}
260+
],
261+
"preload": false,
262+
"refresh": "5m",
263+
"schemaVersion": 41,
264+
"tags": ["q_dev", "copilot", "kiro", "comparison"],
265+
"templating": {
266+
"list": [
267+
{
268+
"current": {},
269+
"datasource": "mysql",
270+
"definition": "SELECT DISTINCT connection_id FROM _tool_copilot_scopes ORDER BY connection_id DESC",
271+
"hide": 0,
272+
"label": "Copilot Connection",
273+
"name": "copilot_connection_id",
274+
"options": [],
275+
"query": "SELECT DISTINCT connection_id FROM _tool_copilot_scopes ORDER BY connection_id DESC",
276+
"refresh": 1,
277+
"type": "query"
278+
},
279+
{
280+
"current": {},
281+
"datasource": "mysql",
282+
"definition": "SELECT DISTINCT id FROM _tool_copilot_scopes WHERE connection_id = CAST('${copilot_connection_id}' AS UNSIGNED)",
283+
"hide": 0,
284+
"label": "Copilot Scope",
285+
"name": "copilot_scope_id",
286+
"options": [],
287+
"query": "SELECT DISTINCT id FROM _tool_copilot_scopes WHERE connection_id = CAST('${copilot_connection_id}' AS UNSIGNED)",
288+
"refresh": 1,
289+
"type": "query"
290+
}
291+
]
292+
},
293+
"time": { "from": "now-90d", "to": "now" },
294+
"timepicker": {},
295+
"timezone": "utc",
296+
"title": "Multi-AI Tool Comparison",
297+
"uid": "multi_ai_comparison",
298+
"version": 1
299+
}

0 commit comments

Comments
 (0)