Skip to content

Commit f067fd4

Browse files
warren830tamas-la
warren830
authored and
tamas-la
committed
feat(q-dev): add Steering & Spec Mode Adoption dashboard (#8798)
Track Kiro steering rules and spec mode adoption: - User/request adoption rate stats - Weekly adoption rate trend - Steering impact on prompt/response length - Per-user feature adoption table
1 parent 4d987b8 commit f067fd4

1 file changed

Lines changed: 212 additions & 0 deletions

File tree

‎grafana/dashboards/SteeringAdoptionTracker.json‎

Lines changed: 212 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,212 @@
1+
{
2+
"annotations": {
3+
"list": [
4+
{
5+
"builtIn": 1,
6+
"datasource": "-- Grafana --",
7+
"enable": true,
8+
"hide": true,
9+
"iconColor": "rgba(0, 211, 255, 1)",
10+
"name": "Annotations & Alerts",
11+
"type": "dashboard"
12+
}
13+
]
14+
},
15+
"editable": true,
16+
"fiscalYearStartMonth": 0,
17+
"graphTooltip": 1,
18+
"id": null,
19+
"links": [],
20+
"panels": [
21+
{
22+
"datasource": "mysql",
23+
"fieldConfig": {
24+
"defaults": {
25+
"color": { "mode": "thresholds" },
26+
"mappings": [],
27+
"thresholds": { "mode": "absolute", "steps": [{ "color": "blue" }] },
28+
"unit": "percent"
29+
},
30+
"overrides": []
31+
},
32+
"gridPos": { "h": 6, "w": 8, "x": 0, "y": 0 },
33+
"id": 1,
34+
"options": {
35+
"colorMode": "value", "graphMode": "area",
36+
"reduceOptions": { "calcs": ["sum"], "fields": "", "values": false },
37+
"textMode": "auto"
38+
},
39+
"targets": [
40+
{
41+
"datasource": "mysql",
42+
"format": "table",
43+
"rawQuery": true,
44+
"rawSql": "SELECT ROUND(COUNT(DISTINCT CASE WHEN has_steering = 1 THEN user_id END) * 100.0 / NULLIF(COUNT(DISTINCT user_id), 0), 0) AS 'value'\nFROM _tool_q_dev_chat_log WHERE $__timeFilter(timestamp)",
45+
"refId": "A"
46+
}
47+
],
48+
"title": "Users with Steering",
49+
"type": "stat"
50+
},
51+
{
52+
"datasource": "mysql",
53+
"fieldConfig": {
54+
"defaults": {
55+
"color": { "mode": "thresholds" },
56+
"mappings": [],
57+
"thresholds": { "mode": "absolute", "steps": [{ "color": "purple" }] },
58+
"unit": "percent"
59+
},
60+
"overrides": []
61+
},
62+
"gridPos": { "h": 6, "w": 8, "x": 8, "y": 0 },
63+
"id": 2,
64+
"options": {
65+
"colorMode": "value", "graphMode": "area",
66+
"reduceOptions": { "calcs": ["sum"], "fields": "", "values": false },
67+
"textMode": "auto"
68+
},
69+
"targets": [
70+
{
71+
"datasource": "mysql",
72+
"format": "table",
73+
"rawQuery": true,
74+
"rawSql": "SELECT ROUND(COUNT(DISTINCT CASE WHEN is_spec_mode = 1 THEN user_id END) * 100.0 / NULLIF(COUNT(DISTINCT user_id), 0), 0) AS 'value'\nFROM _tool_q_dev_chat_log WHERE $__timeFilter(timestamp)",
75+
"refId": "A"
76+
}
77+
],
78+
"title": "Users with Spec Mode",
79+
"type": "stat"
80+
},
81+
{
82+
"datasource": "mysql",
83+
"fieldConfig": {
84+
"defaults": {
85+
"color": { "mode": "thresholds" },
86+
"mappings": [],
87+
"thresholds": { "mode": "absolute", "steps": [{ "color": "green" }] },
88+
"unit": "percent"
89+
},
90+
"overrides": []
91+
},
92+
"gridPos": { "h": 6, "w": 8, "x": 16, "y": 0 },
93+
"id": 3,
94+
"options": {
95+
"colorMode": "value", "graphMode": "area",
96+
"reduceOptions": { "calcs": ["sum"], "fields": "", "values": false },
97+
"textMode": "auto"
98+
},
99+
"targets": [
100+
{
101+
"datasource": "mysql",
102+
"format": "table",
103+
"rawQuery": true,
104+
"rawSql": "SELECT ROUND(SUM(CASE WHEN has_steering = 1 THEN 1 ELSE 0 END) * 100.0 / COUNT(*), 1) AS 'value'\nFROM _tool_q_dev_chat_log WHERE $__timeFilter(timestamp)",
105+
"refId": "A"
106+
}
107+
],
108+
"title": "Requests with Steering",
109+
"type": "stat"
110+
},
111+
{
112+
"datasource": "mysql",
113+
"description": "Weekly trend of steering and spec mode adoption rate",
114+
"fieldConfig": {
115+
"defaults": {
116+
"color": { "mode": "palette-classic" },
117+
"custom": {
118+
"drawStyle": "line", "fillOpacity": 10, "lineInterpolation": "smooth", "lineWidth": 2,
119+
"showPoints": "never", "spanNulls": true,
120+
"stacking": { "mode": "none" }, "thresholdsStyle": { "mode": "off" }
121+
},
122+
"unit": "percentunit"
123+
},
124+
"overrides": []
125+
},
126+
"gridPos": { "h": 8, "w": 24, "x": 0, "y": 6 },
127+
"id": 4,
128+
"options": {
129+
"legend": { "calcs": ["mean", "max"], "displayMode": "table", "placement": "right", "showLegend": true },
130+
"tooltip": { "mode": "multi" }
131+
},
132+
"targets": [
133+
{
134+
"datasource": "mysql",
135+
"format": "time_series",
136+
"rawQuery": true,
137+
"rawSql": "SELECT\n DATE_SUB(DATE(timestamp), INTERVAL WEEKDAY(DATE(timestamp)) DAY) AS time,\n SUM(CASE WHEN has_steering = 1 THEN 1 ELSE 0 END) / COUNT(*) AS 'Steering Rate',\n SUM(CASE WHEN is_spec_mode = 1 THEN 1 ELSE 0 END) / COUNT(*) AS 'Spec Mode Rate'\nFROM _tool_q_dev_chat_log\nWHERE $__timeFilter(timestamp)\nGROUP BY DATE_SUB(DATE(timestamp), INTERVAL WEEKDAY(DATE(timestamp)) DAY)\nORDER BY time",
138+
"refId": "A"
139+
}
140+
],
141+
"title": "Weekly Steering & Spec Mode Adoption Rate",
142+
"type": "timeseries"
143+
},
144+
{
145+
"datasource": "mysql",
146+
"description": "Compare prompt and response quality between steering and non-steering sessions",
147+
"fieldConfig": {
148+
"defaults": {
149+
"color": { "mode": "palette-classic" },
150+
"thresholds": { "mode": "absolute", "steps": [{ "color": "green" }] },
151+
"unit": "short"
152+
},
153+
"overrides": []
154+
},
155+
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 14 },
156+
"id": 5,
157+
"options": {
158+
"barRadius": 0.1, "barWidth": 0.5,
159+
"orientation": "horizontal", "showValue": "auto", "stacking": "none",
160+
"tooltip": { "mode": "single" }
161+
},
162+
"targets": [
163+
{
164+
"datasource": "mysql",
165+
"format": "table",
166+
"rawQuery": true,
167+
"rawSql": "SELECT 'With Steering' AS 'Mode', ROUND(AVG(prompt_length)) AS 'Avg Prompt', ROUND(AVG(response_length)) AS 'Avg Response'\nFROM _tool_q_dev_chat_log WHERE has_steering = 1 AND $__timeFilter(timestamp)\nUNION ALL\nSELECT 'Without Steering', ROUND(AVG(prompt_length)), ROUND(AVG(response_length))\nFROM _tool_q_dev_chat_log WHERE has_steering = 0 AND $__timeFilter(timestamp)",
168+
"refId": "A"
169+
}
170+
],
171+
"title": "Steering Impact: Prompt & Response Length",
172+
"type": "bargauge"
173+
},
174+
{
175+
"datasource": "mysql",
176+
"description": "Per-user steering and spec mode usage",
177+
"fieldConfig": {
178+
"defaults": {
179+
"color": { "mode": "thresholds" },
180+
"custom": { "align": "auto", "cellOptions": { "type": "auto" }, "filterable": true },
181+
"thresholds": { "mode": "absolute", "steps": [{ "color": "green" }] }
182+
},
183+
"overrides": []
184+
},
185+
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 14 },
186+
"id": 6,
187+
"options": { "cellHeight": "sm", "showHeader": true },
188+
"targets": [
189+
{
190+
"datasource": "mysql",
191+
"format": "table",
192+
"rawQuery": true,
193+
"rawSql": "SELECT\n COALESCE(MAX(display_name), user_id) AS 'User',\n COUNT(*) AS 'Total Chats',\n SUM(CASE WHEN has_steering = 1 THEN 1 ELSE 0 END) AS 'Steering',\n SUM(CASE WHEN is_spec_mode = 1 THEN 1 ELSE 0 END) AS 'Spec Mode',\n ROUND(SUM(CASE WHEN has_steering = 1 THEN 1 ELSE 0 END) * 100.0 / COUNT(*), 1) AS 'Steering %'\nFROM _tool_q_dev_chat_log\nWHERE $__timeFilter(timestamp)\nGROUP BY user_id\nORDER BY COUNT(*) DESC",
194+
"refId": "A"
195+
}
196+
],
197+
"title": "Per-User Feature Adoption",
198+
"type": "table"
199+
}
200+
],
201+
"preload": false,
202+
"refresh": "5m",
203+
"schemaVersion": 41,
204+
"tags": ["q_dev", "kiro", "steering", "adoption"],
205+
"templating": { "list": [] },
206+
"time": { "from": "now-90d", "to": "now" },
207+
"timepicker": {},
208+
"timezone": "utc",
209+
"title": "Kiro Steering & Spec Mode Adoption",
210+
"uid": "kiro_steering_adoption",
211+
"version": 1
212+
}

0 commit comments

Comments
 (0)