Skip to content

Commit 85d6909

Browse files
JanKrivanekJanKrivanek
authored
Add static eval via skill-validator (#1195)
* Add static eval via skill-validator * Add issues: write permission for PR comment posting
1 parent 819a8fa commit 85d6909

2 files changed

Lines changed: 537 additions & 0 deletions

File tree

‎.github/workflows/skill-check.yml‎

Lines changed: 217 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,217 @@
1+
name: Skill Validator — PR Gate
2+
3+
on:
4+
pull_request:
5+
branches: [staged]
6+
types: [opened, synchronize, reopened]
7+
paths:
8+
- "skills/**"
9+
- "agents/**"
10+
- "plugins/**/skills/**"
11+
- "plugins/**/agents/**"
12+
13+
permissions:
14+
contents: read
15+
pull-requests: write
16+
issues: write
17+
18+
jobs:
19+
skill-check:
20+
runs-on: ubuntu-latest
21+
steps:
22+
- name: Checkout code
23+
uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1
24+
with:
25+
fetch-depth: 0
26+
27+
# ── Download & cache skill-validator ──────────────────────────
28+
- name: Get cache key date
29+
id: cache-date
30+
run: echo "date=$(date +%Y-%m-%d)" >> "$GITHUB_OUTPUT"
31+
32+
- name: Restore skill-validator from cache
33+
id: cache-sv
34+
uses: actions/cache/restore@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5.0.3
35+
with:
36+
path: .skill-validator
37+
key: skill-validator-linux-x64-${{ steps.cache-date.outputs.date }}
38+
restore-keys: |
39+
skill-validator-linux-x64-
40+
41+
- name: Download skill-validator
42+
if: steps.cache-sv.outputs.cache-hit != 'true'
43+
run: |
44+
mkdir -p .skill-validator
45+
curl -fsSL \
46+
"https://github.com/dotnet/skills/releases/download/skill-validator-nightly/skill-validator-linux-x64.tar.gz" \
47+
-o .skill-validator/skill-validator-linux-x64.tar.gz
48+
tar -xzf .skill-validator/skill-validator-linux-x64.tar.gz -C .skill-validator
49+
rm .skill-validator/skill-validator-linux-x64.tar.gz
50+
chmod +x .skill-validator/skill-validator
51+
52+
- name: Save skill-validator to cache
53+
if: steps.cache-sv.outputs.cache-hit != 'true'
54+
uses: actions/cache/save@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5.0.3
55+
with:
56+
path: .skill-validator
57+
key: skill-validator-linux-x64-${{ steps.cache-date.outputs.date }}
58+
59+
# ── Detect changed skills & agents ────────────────────────────
60+
- name: Detect changed skills and agents
61+
id: detect
62+
run: |
63+
CHANGED_FILES=$(git diff --name-only origin/${{ github.base_ref }}...HEAD)
64+
65+
# Extract unique skill directories that were touched
66+
SKILL_DIRS=$(echo "$CHANGED_FILES" | grep -oP '^skills/[^/]+' | sort -u || true)
67+
68+
# Extract agent files that were touched
69+
AGENT_FILES=$(echo "$CHANGED_FILES" | grep -oP '^agents/[^/]+\.agent\.md$' | sort -u || true)
70+
71+
# Extract plugin skill directories
72+
PLUGIN_SKILL_DIRS=$(echo "$CHANGED_FILES" | grep -oP '^plugins/[^/]+/skills/[^/]+' | sort -u || true)
73+
74+
# Extract plugin agent files
75+
PLUGIN_AGENT_FILES=$(echo "$CHANGED_FILES" | grep -oP '^plugins/[^/]+/agents/[^/]+\.agent\.md$' | sort -u || true)
76+
77+
# Build CLI arguments for --skills
78+
SKILL_ARGS=""
79+
for dir in $SKILL_DIRS $PLUGIN_SKILL_DIRS; do
80+
if [ -d "$dir" ]; then
81+
SKILL_ARGS="$SKILL_ARGS $dir"
82+
fi
83+
done
84+
85+
# Build CLI arguments for --agents
86+
AGENT_ARGS=""
87+
for f in $AGENT_FILES $PLUGIN_AGENT_FILES; do
88+
if [ -f "$f" ]; then
89+
AGENT_ARGS="$AGENT_ARGS $f"
90+
fi
91+
done
92+
93+
SKILL_COUNT=$(echo "$SKILL_ARGS" | xargs -n1 2>/dev/null | wc -l || echo 0)
94+
AGENT_COUNT=$(echo "$AGENT_ARGS" | xargs -n1 2>/dev/null | wc -l || echo 0)
95+
TOTAL=$((SKILL_COUNT + AGENT_COUNT))
96+
97+
echo "skill_args=$SKILL_ARGS" >> "$GITHUB_OUTPUT"
98+
echo "agent_args=$AGENT_ARGS" >> "$GITHUB_OUTPUT"
99+
echo "total=$TOTAL" >> "$GITHUB_OUTPUT"
100+
echo "skill_count=$SKILL_COUNT" >> "$GITHUB_OUTPUT"
101+
echo "agent_count=$AGENT_COUNT" >> "$GITHUB_OUTPUT"
102+
103+
echo "Found $SKILL_COUNT skill dir(s) and $AGENT_COUNT agent file(s) to check."
104+
105+
# ── Run skill-validator check ─────────────────────────────────
106+
- name: Run skill-validator check
107+
id: check
108+
if: steps.detect.outputs.total != '0'
109+
run: |
110+
SKILL_ARGS="${{ steps.detect.outputs.skill_args }}"
111+
AGENT_ARGS="${{ steps.detect.outputs.agent_args }}"
112+
113+
CMD=".skill-validator/skill-validator check --verbose"
114+
115+
if [ -n "$SKILL_ARGS" ]; then
116+
CMD="$CMD --skills $SKILL_ARGS"
117+
fi
118+
119+
if [ -n "$AGENT_ARGS" ]; then
120+
CMD="$CMD --agents $AGENT_ARGS"
121+
fi
122+
123+
echo "Running: $CMD"
124+
125+
# Capture output; don't fail the workflow (warn-only mode)
126+
set +e
127+
OUTPUT=$($CMD 2>&1)
128+
EXIT_CODE=$?
129+
set -e
130+
131+
echo "exit_code=$EXIT_CODE" >> "$GITHUB_OUTPUT"
132+
133+
# Save output to file (multi-line safe)
134+
echo "$OUTPUT" > sv-output.txt
135+
136+
echo "$OUTPUT"
137+
138+
# ── Post / update PR comment ──────────────────────────────────
139+
- name: Post PR comment with results
140+
if: steps.detect.outputs.total != '0'
141+
uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7.1.0
142+
with:
143+
script: |
144+
const fs = require('fs');
145+
146+
const marker = '<!-- skill-validator-results -->';
147+
const output = fs.readFileSync('sv-output.txt', 'utf8').trim();
148+
const exitCode = '${{ steps.check.outputs.exit_code }}';
149+
const skillCount = parseInt('${{ steps.detect.outputs.skill_count }}', 10);
150+
const agentCount = parseInt('${{ steps.detect.outputs.agent_count }}', 10);
151+
const totalChecked = skillCount + agentCount;
152+
153+
// Count errors, warnings, advisories from output
154+
const errorCount = (output.match(/\bError\b/gi) || []).length;
155+
const warningCount = (output.match(/\bWarning\b/gi) || []).length;
156+
const advisoryCount = (output.match(/\bAdvisory\b/gi) || []).length;
157+
158+
let statusLine;
159+
if (errorCount > 0) {
160+
statusLine = `**${totalChecked} resource(s) checked** | ⛔ ${errorCount} error(s) | ⚠️ ${warningCount} warning(s) | ℹ️ ${advisoryCount} advisory(ies)`;
161+
} else if (warningCount > 0) {
162+
statusLine = `**${totalChecked} resource(s) checked** | ⚠️ ${warningCount} warning(s) | ℹ️ ${advisoryCount} advisory(ies)`;
163+
} else {
164+
statusLine = `**${totalChecked} resource(s) checked** | ✅ All checks passed`;
165+
}
166+
167+
const body = [
168+
marker,
169+
'## 🔍 Skill Validator Results',
170+
'',
171+
statusLine,
172+
'',
173+
'<details>',
174+
'<summary>Full output</summary>',
175+
'',
176+
'```',
177+
output,
178+
'```',
179+
'',
180+
'</details>',
181+
'',
182+
exitCode !== '0'
183+
? '> **Note:** Errors were found. These are currently reported as warnings and do not block merge. Please review and address when possible.'
184+
: '',
185+
].join('\n');
186+
187+
// Find existing comment with our marker
188+
const { data: comments } = await github.rest.issues.listComments({
189+
owner: context.repo.owner,
190+
repo: context.repo.repo,
191+
issue_number: context.issue.number,
192+
per_page: 100,
193+
});
194+
195+
const existing = comments.find(c => c.body.includes(marker));
196+
197+
if (existing) {
198+
await github.rest.issues.updateComment({
199+
owner: context.repo.owner,
200+
repo: context.repo.repo,
201+
comment_id: existing.id,
202+
body,
203+
});
204+
console.log(`Updated existing comment ${existing.id}`);
205+
} else {
206+
await github.rest.issues.createComment({
207+
owner: context.repo.owner,
208+
repo: context.repo.repo,
209+
issue_number: context.issue.number,
210+
body,
211+
});
212+
console.log('Created new PR comment');
213+
}
214+
215+
- name: Post skip notice if no skills changed
216+
if: steps.detect.outputs.total == '0'
217+
run: echo "No skill or agent files changed in this PR — skipping validation."

0 commit comments

Comments
 (0)