From 860a7f5be8f0681fa82638f3537554f27f106e2d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Sat, 4 Jul 2026 18:15:14 +0200 Subject: [PATCH 1/2] test: slow-test ratchet, budget-derived emulator poll, speed guidance from experiments MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Measured (2026-07-04, full unit suite: 340 files / 3,210 tests / 48s wall): wall clock was bounded by the slowest FILE (44.6s android monolith at ~7x file-level parallelism), and the slowest tests were sleeping through real production budgets (10.8s proving 'times out' by waiting the constant out, 8s emulator polls at 1Hz, real retry backoff). Two config experiments rejected with data: --no-isolate exploded the suite to 205s (module state thrashes across files sharing workers) and --pool=threads changed nothing. - scripts/vitest-slow-test-reporter.ts: the slow-test ratchet. Unit budget 2.5s / integration 15s; failure at 2x budget (the band between reports without failing so host-load variance cannot make the gate cry wolf); 36 pinned offenders, exact keys, ratchet-only pin (tracking #1098). - waitForAndroidEmulatorByAvdName: poll cadence derives from the caller's budget (min 1s, floor 50ms, ~timeout/20) — devices.test.ts 25.6s -> 2.8s (9x) in isolation, and short-budget production calls stop sampling at 1Hz against small budgets. - vitest.config: slowTestThreshold 500 for local visibility; reporter wired; isolation/pool decisions documented with the measurements. - docs/agents/testing.md 'Speed rules' + AGENTS.md testing bullet: the three conversion patterns in preference order (budget-derived cadence, budget-wiring assertion, fake clocks), the no-seam constraint, and the file-granularity Amdahl argument that makes the monolith test split a wall-clock fix, not just navigation. --- AGENTS.md | 1 + docs/agents/testing.md | 29 +++++++ scripts/vitest-slow-test-reporter.ts | 121 +++++++++++++++++++++++++++ src/platforms/android/devices.ts | 10 ++- vitest.config.ts | 9 ++ 5 files changed, 169 insertions(+), 1 deletion(-) create mode 100644 scripts/vitest-slow-test-reporter.ts diff --git a/AGENTS.md b/AGENTS.md index 62066e7a8..63922dba4 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -36,6 +36,7 @@ Single-context repo. Read `CONTEXT.md` for domain language and testing/architect - Ship the minimum code that solves the problem: no speculative features, no single-use abstractions, and no unrelated cleanup. - Match existing style. Remove imports/variables your change made unused. - Test through public interfaces when possible. Do not add unrelated exports just to make tests easier. +- Unit tests never wait real time: inject the budget, derive the cadence from it, or assert the budget is wired — the slow-test ratchet (`scripts/vitest-slow-test-reporter.ts`) fails tests past 2x budget; speed rules and conversion patterns in `docs/agents/testing.md`. - Prefer type-level checks when TypeScript can enforce a contract or invalid shape. - Use `unknown` only at trust boundaries: parsed JSON, daemon/runtime payloads, catch values, generic I/O, or parser callbacks. Once a value is validated or its producer has a known contract, narrow to a domain type or focused parser/helper instead of carrying `unknown` through internal helper and formatter signatures. - Keep modules small for agent context safety: diff --git a/docs/agents/testing.md b/docs/agents/testing.md index 130cc0594..b36913532 100644 --- a/docs/agents/testing.md +++ b/docs/agents/testing.md @@ -9,3 +9,32 @@ AGENT_DEVICE_WEB_E2E=1 pnpm test:smoke:web ``` The test is skipped unless `AGENT_DEVICE_WEB_E2E=1` is set. The test runs `agent-device web setup` and `agent-device web doctor` with an isolated state directory before opening the fixture URL, so it verifies the public managed-backend setup path instead of relying on a global `agent-browser`. CI runs the lane on Node 24 because the managed backend requires Node >= 24. Failure artifacts, daemon state, and browser config are written under `test/artifacts/web/`. + +## Speed rules (experiment-backed, 2026-07-04) + +Measured on the full unit suite (340 files, 3,210 tests, 48s wall at ~7x parallelism): + +- **Wall clock equals the slowest file.** The 44.6s android monolith bounded the whole 48s run + (Amdahl at file granularity: vitest parallelizes per file). Splitting monolith test files is a + wall-clock optimization, not just a navigation one — see the AGENTS.md test-topology mirror rule. +- **Unit tests must not wait real time.** The suite's worst tests slept through production budgets: + 10.8s to prove "times out" by waiting out the full constant, 8s emulator-boot polls at 1Hz, real + retry backoffs. Conversion patterns, in preference order (tracking issue #1098): + 1. *Budget-derived cadence* (production-legit): poll intervals scale with the caller's timeout — + this took `devices.test.ts` from 25.6s to 2.8s (9x) while making short-budget production calls + more responsive. + 2. *Budget-wiring assertion*: don't re-prove the exec layer's timeout per call site; mock the tool + layer and assert the right `timeoutMs` constant is passed. Exec-layer timeout semantics are + proven once, in exec's own tests. + 3. *Fake clocks* where the code accepts an injected clock. + Never add a test-only DI seam for this — the CI gate forbids it; patterns 1–2 are production + improvements and test restructurings respectively. +- **The slow-test ratchet** (`scripts/vitest-slow-test-reporter.ts`) enforces this: unit budget + 2.5s, integration 15s, failure at 2x budget (the band between reports without failing — host + load legitimately stretches borderline tests, and a flaky gate trains people to ignore it). + The pin list only shrinks, or grows in the same PR with a justification. +- **Isolation stays ON; pool stays forks — both measured.** `--no-isolate`: 205s wall vs 48s + (module state — timers, memos, singletons — thrashes across files sharing a worker). + `--pool=threads`: no change (50.4s). The ~100s aggregate import overhead is the price of + isolation and is paid in parallel; reduce it per file by importing the module under test, not + platform barrels. diff --git a/scripts/vitest-slow-test-reporter.ts b/scripts/vitest-slow-test-reporter.ts new file mode 100644 index 000000000..fdd70f945 --- /dev/null +++ b/scripts/vitest-slow-test-reporter.ts @@ -0,0 +1,121 @@ +import type { Reporter, TestCase, TestModule } from 'vitest/node'; + +/** + * Slow-test ratchet (see docs/agents/testing.md "Speed rules"). + * + * Unit tests must not wait real time: measured 2026-07-04, the unit suite's + * wall clock was bounded by files whose tests slept through production + * timeouts (a 10.8s test proving "times out" by waiting the full 10s budget). + * This reporter fails the run when a unit test exceeds the budget UNLESS it + * is pinned below. The pin is a ratchet: it may only shrink, or grow in the + * same PR that adds the entry here with a justification — the same + * only-changes-in-reviewed-diffs rule as the guarantee-matrix gap list. + * + * Budgets are per suite family: integration scenarios drive a real daemon + * request path and get more room; unit tests get 2.5s, which is already + * generous for injected-time tests. + */ +const UNIT_BUDGET_MS = 2_500; +const INTEGRATION_BUDGET_MS = 15_000; +// Enforcement fires at 2x budget: host load legitimately stretches a +// borderline test by tens of percent, and a wall-clock gate that flakes under +// contention trains people to ignore it. Between budget and 2x budget the +// gate reports without failing. +const ENFORCE_FACTOR = 2; + +// Ratchet pin: known offenders at gate introduction (tracking issue #1098). +// Every entry is a test that waits real time (production timeout constants, +// 1Hz poll loops, real retry backoff) and shrinks as those are converted to +// budget-wiring assertions or budget-derived poll intervals. +const PINNED_SLOW_UNIT_TESTS = new Set([ + "src/__tests__/daemon-entrypoint.test.ts :: daemon runtime starts HTTP transport in-process and shuts down cleanly", + "src/daemon/__tests__/artifact-materialization.test.ts :: materializeArtifact extracts iOS app bundle tar archives and returns the installable .app path", + "src/daemon/__tests__/runtime-hints.test.ts :: applyRuntimeHintsToApp preserves write failures after a successful run-as probe", + "src/daemon/__tests__/runtime-hints.test.ts :: applyRuntimeHintsToApp writes React Native Android dev prefs", + "src/daemon/__tests__/runtime-hints.test.ts :: applyRuntimeHintsToApp writes iOS simulator React Native defaults", + "src/daemon/__tests__/runtime-hints.test.ts :: clearRuntimeHintsFromApp removes managed Android runtime prefs but preserves unrelated entries", + "src/daemon/handlers/__tests__/session-replay-vars.test.ts :: runReplayScriptFile skips Maestro runFlow.when.visible commands when absent", + "src/platforms/__tests__/install-source.test.ts :: prepareIosInstallArtifact cleans URL materialization when IPA payload resolution fails", + "src/platforms/__tests__/install-source.test.ts :: prepareIosInstallArtifact extracts trusted GitHub artifact ZIP containing nested app tar", + "src/platforms/__tests__/install-source.test.ts :: prepareIosInstallArtifact extracts trusted GitHub artifact ZIP containing one IPA", + "src/platforms/android/__tests__/devices.test.ts :: ensureAndroidEmulatorBooted falls back to ANDROID_SDK_ROOT when PATH is incomplete", + "src/platforms/android/__tests__/devices.test.ts :: ensureAndroidEmulatorBooted launches emulator in headless mode when requested", + "src/platforms/android/__tests__/devices.test.ts :: ensureAndroidEmulatorBooted launches emulator with GUI by default", + "src/platforms/android/__tests__/devices.test.ts :: ensureAndroidEmulatorBooted reuses running emulator for headless requests", + "src/platforms/android/__tests__/devices.test.ts :: listAndroidDevices falls back to model when emulator avd name is unavailable", + "src/platforms/android/__tests__/index.test.ts :: fillAndroid uses chunk-safe shell input and retries when verification still fails", + "src/platforms/android/__tests__/index.test.ts :: installAndroidApp .aab reports missing bundletool tooling", + "src/platforms/android/__tests__/index.test.ts :: installAndroidApp installs .aab via bundletool build-apks + install-apks", + "src/platforms/android/__tests__/index.test.ts :: installAndroidApp installs .apk via adb install -r", + "src/platforms/android/__tests__/index.test.ts :: installAndroidApp resolves packageName and launchTarget from nested archive artifacts", + "src/platforms/android/__tests__/perf.test.ts :: stopAndroidSimpleperfProfile fails before pull when remote artifact never stabilizes", + "src/platforms/android/__tests__/snapshot-helper-session.test.ts :: allows a persistent session snapshot to use the helper command budget", + "src/platforms/apple/core/__tests__/index.test.ts :: openIosSimulatorApp times out instead of hanging indefinitely", + "src/platforms/apple/core/__tests__/index.test.ts :: prepareSimulatorStatusBarForScreenshot restores prior visible overrides", + "src/platforms/apple/core/__tests__/index.test.ts :: prepareSimulatorStatusBarForScreenshot skips known redundant status bar commands", + "src/platforms/apple/core/__tests__/index.test.ts :: prepareSimulatorStatusBarForScreenshot still normalizes when snapshotting current overrides fails", + "src/platforms/apple/core/__tests__/index.test.ts :: screenshotIos retries simulator capture timeouts and eventually succeeds", + "src/platforms/apple/core/__tests__/runner-adoption.test.ts :: adoption is skipped on artifact fingerprint mismatch", + "src/platforms/apple/core/__tests__/runner-adoption.test.ts :: adoption is skipped when the probe fails", + "src/platforms/apple/core/__tests__/runner-client.test.ts :: ensureXctestrun falls back to scan when cache manifest is stale", + "src/platforms/apple/core/__tests__/runner-client.test.ts :: ensureXctestrun rebuilds after cached macOS runner repair failure", + "src/platforms/apple/core/__tests__/runner-client.test.ts :: ensureXctestrun rebuilds cached runner when Swift build flags mismatch", + "src/platforms/apple/core/__tests__/runner-client.test.ts :: ensureXctestrun rebuilds foreign artifacts when metadata does not match", + "src/platforms/apple/core/__tests__/runner-xctestrun.test.ts :: setup metadata script matches expected iOS simulator cache metadata", + "src/recording/__tests__/recording-scripts.test.ts :: recording overlay Swift script typechecks", + "src/utils/__tests__/daemon-client.test.ts :: cleanupFailedDaemonStartupMetadata retains live startup daemon on timeout", +]); + +type Offender = { key: string; durationMs: number; budgetMs: number; enforce: boolean }; + +export default class SlowTestGateReporter implements Reporter { + private offenders: Offender[] = []; + private root = ''; + + onInit(ctx: { config: { root: string } }): void { + this.root = ctx.config.root; + } + + onTestCaseResult(testCase: TestCase): void { + const result = testCase.result(); + if (result.state !== 'passed' && result.state !== 'failed') return; + const diagnostic = testCase.diagnostic(); + const durationMs = diagnostic?.duration ?? 0; + const moduleId = (testCase.module as TestModule).moduleId; + const relative = this.root && moduleId.startsWith(this.root) + ? moduleId.slice(this.root.length + 1) + : moduleId; + const budgetMs = relative.startsWith('src/') ? UNIT_BUDGET_MS : INTEGRATION_BUDGET_MS; + if (durationMs <= budgetMs) return; + const key = `${relative} :: ${testCase.fullName.split(' > ').join(' ')}`; + const pinKey = `${relative} :: ${testCase.name}`; + if (PINNED_SLOW_UNIT_TESTS.has(pinKey) || PINNED_SLOW_UNIT_TESTS.has(key)) return; + this.offenders.push({ key, durationMs, budgetMs, enforce: durationMs > budgetMs * ENFORCE_FACTOR }); + } + + onTestRunEnd(): void { + if (this.offenders.length === 0) return; + const sorted = this.offenders.sort((a, b) => b.durationMs - a.durationMs); + const line = (o: Offender): string => + ` ${(o.durationMs / 1000).toFixed(2)}s (budget ${o.budgetMs / 1000}s) ${o.key}`; + const failing = sorted.filter((o) => o.enforce); + const warning = sorted.filter((o) => !o.enforce); + if (warning.length > 0) { + // eslint-disable-next-line no-console + console.error( + `\nSlow-test gate: ${warning.length} test(s) over budget (within the 2x load-variance band, not failing):\n` + + warning.map(line).join('\n'), + ); + } + if (failing.length === 0) return; + // eslint-disable-next-line no-console + console.error( + `\nSlow-test gate: ${failing.length} test(s) exceeded ${ENFORCE_FACTOR}x the wall-clock budget.\n` + + `Tests must not wait real time — inject the timeout/poll budget or assert the budget is\n` + + `wired instead of waiting it out (docs/agents/testing.md). If the wait is genuinely\n` + + `irreducible, pin it in scripts/vitest-slow-test-reporter.ts in this PR with a reason.\n` + + failing.map(line).join('\n'), + ); + process.exitCode = 1; + } +} diff --git a/src/platforms/android/devices.ts b/src/platforms/android/devices.ts index cadc6ad56..8a2805a7c 100644 --- a/src/platforms/android/devices.ts +++ b/src/platforms/android/devices.ts @@ -366,6 +366,14 @@ async function waitForAndroidEmulatorByAvdName(params: { timeoutMs: number; }): Promise { const startedAt = Date.now(); + // Poll cadence scales with the caller's budget: a 1Hz sample rate against a + // small budget wastes most of it between checks (a 5s budget deserves 250ms + // sampling), while large budgets keep the gentle 1s cadence. Floor of 50ms + // keeps tight budgets from busy-spinning. + const pollMs = Math.min( + ANDROID_EMULATOR_BOOT_POLL_MS, + Math.max(50, Math.floor(params.timeoutMs / 20)), + ); while (Date.now() - startedAt < params.timeoutMs) { try { const serial = await findAndroidEmulatorSerialByAvdName(params.avdName, params.serial); @@ -382,7 +390,7 @@ async function waitForAndroidEmulatorByAvdName(params: { } catch { // Best-effort polling while adb/emulator process settles. } - await sleep(ANDROID_EMULATOR_BOOT_POLL_MS); + await sleep(pollMs); } throw new AppError('COMMAND_FAILED', 'Android emulator did not appear in time', { avdName: params.avdName, diff --git a/vitest.config.ts b/vitest.config.ts index daaf03c07..4df255ea4 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -2,6 +2,15 @@ import { defineConfig } from 'vitest/config'; export default defineConfig({ test: { + // Wall-clock discipline: unit tests must not wait real time. Measured + // 2026-07-04: the suite's duration was bounded by files sleeping through + // production timeout budgets. slowTestThreshold surfaces creep in local + // output; the slow-test reporter enforces the ratchet (pinned offenders + // only shrink). Isolation stays ON and pool stays forks: measured + // --no-isolate = 205s wall vs 48s (module state thrashes across files), + // threads = no change. + slowTestThreshold: 500, + reporters: ['default', './scripts/vitest-slow-test-reporter.ts'], projects: [ { test: { From bf0e36c30d1017a98b292065acfc25f3b1b95e0a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Sat, 4 Jul 2026 18:41:08 +0200 Subject: [PATCH 2/2] =?UTF-8?q?fix:=20fallow=20findings=20on=20the=20slow-?= =?UTF-8?q?test=20gate=20=E2=80=94=20import=20edge,=20factory=20reporter,?= =?UTF-8?q?=20unit=20tests?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The string-path reporter wiring read as a dead file (fallow cannot see vitest's reporter loading); the config now imports the factory, making the edge real and type-checked. The class shape tripped the unused-class-members rule (framework callbacks are invisible to reference analysis) — converted to a factory returning the Reporter object, with the classification and rendering logic extracted as pure exported functions. Those functions now carry their own unit tests (budget bands, integration budgets, pin matching, warn-vs-fail rendering), which also grounds the CRAP estimate in real references. Canary re-verified: unpinned 5.2s sleeper fails the run with exit 1; clean runs exit 0. --- scripts/vitest-slow-test-reporter.ts | 126 +++++++++++------- .../vitest-slow-test-reporter.test.ts | 66 +++++++++ vitest.config.ts | 3 +- 3 files changed, 149 insertions(+), 46 deletions(-) create mode 100644 src/__tests__/vitest-slow-test-reporter.test.ts diff --git a/scripts/vitest-slow-test-reporter.ts b/scripts/vitest-slow-test-reporter.ts index fdd70f945..9cf1981e7 100644 --- a/scripts/vitest-slow-test-reporter.ts +++ b/scripts/vitest-slow-test-reporter.ts @@ -68,54 +68,90 @@ const PINNED_SLOW_UNIT_TESTS = new Set([ type Offender = { key: string; durationMs: number; budgetMs: number; enforce: boolean }; -export default class SlowTestGateReporter implements Reporter { - private offenders: Offender[] = []; - private root = ''; +function budgetForPath(relativePath: string): number { + return relativePath.startsWith('src/') ? UNIT_BUDGET_MS : INTEGRATION_BUDGET_MS; +} - onInit(ctx: { config: { root: string } }): void { - this.root = ctx.config.root; - } +/** + * Classify one finished test against its budget. Exported for the unit test — + * the reporter shell below is a thin vitest-callback adapter around this. + */ +export function classifySlowTest(params: { + root: string; + moduleId: string; + name: string; + fullName: string; + durationMs: number; +}): Offender | null { + const relative = + params.root && params.moduleId.startsWith(params.root) + ? params.moduleId.slice(params.root.length + 1) + : params.moduleId; + const budgetMs = budgetForPath(relative); + if (params.durationMs <= budgetMs) return null; + const fullKey = `${relative} :: ${params.fullName.split(' > ').join(' ')}`; + if (PINNED_SLOW_UNIT_TESTS.has(`${relative} :: ${params.name}`)) return null; + if (PINNED_SLOW_UNIT_TESTS.has(fullKey)) return null; + return { + key: fullKey, + durationMs: params.durationMs, + budgetMs, + enforce: params.durationMs > budgetMs * ENFORCE_FACTOR, + }; +} - onTestCaseResult(testCase: TestCase): void { - const result = testCase.result(); - if (result.state !== 'passed' && result.state !== 'failed') return; - const diagnostic = testCase.diagnostic(); - const durationMs = diagnostic?.duration ?? 0; - const moduleId = (testCase.module as TestModule).moduleId; - const relative = this.root && moduleId.startsWith(this.root) - ? moduleId.slice(this.root.length + 1) - : moduleId; - const budgetMs = relative.startsWith('src/') ? UNIT_BUDGET_MS : INTEGRATION_BUDGET_MS; - if (durationMs <= budgetMs) return; - const key = `${relative} :: ${testCase.fullName.split(' > ').join(' ')}`; - const pinKey = `${relative} :: ${testCase.name}`; - if (PINNED_SLOW_UNIT_TESTS.has(pinKey) || PINNED_SLOW_UNIT_TESTS.has(key)) return; - this.offenders.push({ key, durationMs, budgetMs, enforce: durationMs > budgetMs * ENFORCE_FACTOR }); +/** Render the gate outcome; returns true when the run must fail. */ +export function reportSlowTests( + offenders: Offender[], + write: (message: string) => void, +): boolean { + if (offenders.length === 0) return false; + const sorted = [...offenders].sort((a, b) => b.durationMs - a.durationMs); + const line = (o: Offender): string => + ` ${(o.durationMs / 1000).toFixed(2)}s (budget ${o.budgetMs / 1000}s) ${o.key}`; + const failing = sorted.filter((o) => o.enforce); + const warning = sorted.filter((o) => !o.enforce); + if (warning.length > 0) { + write( + `\nSlow-test gate: ${warning.length} test(s) over budget (within the ${ENFORCE_FACTOR}x load-variance band, not failing):\n` + + warning.map(line).join('\n'), + ); } + if (failing.length === 0) return false; + write( + `\nSlow-test gate: ${failing.length} test(s) exceeded ${ENFORCE_FACTOR}x the wall-clock budget.\n` + + `Tests must not wait real time — inject the timeout/poll budget or assert the budget is\n` + + `wired instead of waiting it out (docs/agents/testing.md). If the wait is genuinely\n` + + `irreducible, pin it in scripts/vitest-slow-test-reporter.ts in this PR with a reason.\n` + + failing.map(line).join('\n'), + ); + return true; +} - onTestRunEnd(): void { - if (this.offenders.length === 0) return; - const sorted = this.offenders.sort((a, b) => b.durationMs - a.durationMs); - const line = (o: Offender): string => - ` ${(o.durationMs / 1000).toFixed(2)}s (budget ${o.budgetMs / 1000}s) ${o.key}`; - const failing = sorted.filter((o) => o.enforce); - const warning = sorted.filter((o) => !o.enforce); - if (warning.length > 0) { +export default function slowTestGateReporter(): Reporter { + const offenders: Offender[] = []; + let root = ''; + return { + onInit(ctx: { config: { root: string } }): void { + root = ctx.config.root; + }, + onTestCaseResult(testCase: TestCase): void { + const result = testCase.result(); + if (result.state !== 'passed' && result.state !== 'failed') return; + const offender = classifySlowTest({ + root, + moduleId: (testCase.module as TestModule).moduleId, + name: testCase.name, + fullName: testCase.fullName, + durationMs: testCase.diagnostic()?.duration ?? 0, + }); + if (offender) offenders.push(offender); + }, + onTestRunEnd(): void { // eslint-disable-next-line no-console - console.error( - `\nSlow-test gate: ${warning.length} test(s) over budget (within the 2x load-variance band, not failing):\n` + - warning.map(line).join('\n'), - ); - } - if (failing.length === 0) return; - // eslint-disable-next-line no-console - console.error( - `\nSlow-test gate: ${failing.length} test(s) exceeded ${ENFORCE_FACTOR}x the wall-clock budget.\n` + - `Tests must not wait real time — inject the timeout/poll budget or assert the budget is\n` + - `wired instead of waiting it out (docs/agents/testing.md). If the wait is genuinely\n` + - `irreducible, pin it in scripts/vitest-slow-test-reporter.ts in this PR with a reason.\n` + - failing.map(line).join('\n'), - ); - process.exitCode = 1; - } + if (reportSlowTests(offenders, (message) => console.error(message))) { + process.exitCode = 1; + } + }, + }; } diff --git a/src/__tests__/vitest-slow-test-reporter.test.ts b/src/__tests__/vitest-slow-test-reporter.test.ts new file mode 100644 index 000000000..0fb6412d8 --- /dev/null +++ b/src/__tests__/vitest-slow-test-reporter.test.ts @@ -0,0 +1,66 @@ +import { test } from 'vitest'; +import assert from 'node:assert/strict'; +import { classifySlowTest, reportSlowTests } from '../../scripts/vitest-slow-test-reporter.ts'; + +const base = { + root: '/repo', + moduleId: '/repo/src/utils/__tests__/example.test.ts', + name: 'does a thing', + fullName: 'group > does a thing', +}; + +test('under-budget tests are not offenders', () => { + assert.equal(classifySlowTest({ ...base, durationMs: 2_000 }), null); +}); + +test('over-budget unit tests enter the warn band; 2x budget enforces', () => { + const warn = classifySlowTest({ ...base, durationMs: 3_000 }); + assert.ok(warn); + assert.equal(warn.enforce, false); + const fail = classifySlowTest({ ...base, durationMs: 5_100 }); + assert.ok(fail); + assert.equal(fail.enforce, true); + assert.equal(fail.key, 'src/utils/__tests__/example.test.ts :: group does a thing'); +}); + +test('integration paths get the larger budget', () => { + const offender = classifySlowTest({ + ...base, + moduleId: '/repo/test/integration/provider-scenarios/example.test.ts', + durationMs: 10_000, + }); + assert.equal(offender, null); +}); + +test('pinned tests never become offenders', () => { + const offender = classifySlowTest({ + root: '/repo', + moduleId: '/repo/src/platforms/android/__tests__/index.test.ts', + name: 'installAndroidApp installs .apk via adb install -r', + fullName: 'installAndroidApp installs .apk via adb install -r', + durationMs: 9_000, + }); + assert.equal(offender, null); +}); + +test('reportSlowTests fails only on enforced offenders and prints both bands', () => { + const messages: string[] = []; + const warnOnly = reportSlowTests( + [{ key: 'a', durationMs: 3_000, budgetMs: 2_500, enforce: false }], + (m) => messages.push(m), + ); + assert.equal(warnOnly, false); + assert.match(messages[0] ?? '', /load-variance band/); + + messages.length = 0; + const failing = reportSlowTests( + [ + { key: 'a', durationMs: 3_000, budgetMs: 2_500, enforce: false }, + { key: 'b', durationMs: 6_000, budgetMs: 2_500, enforce: true }, + ], + (m) => messages.push(m), + ); + assert.equal(failing, true); + assert.equal(messages.length, 2); + assert.match(messages[1] ?? '', /must not wait real time/); +}); diff --git a/vitest.config.ts b/vitest.config.ts index 4df255ea4..96128af23 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -1,4 +1,5 @@ import { defineConfig } from 'vitest/config'; +import slowTestGateReporter from './scripts/vitest-slow-test-reporter.ts'; export default defineConfig({ test: { @@ -10,7 +11,7 @@ export default defineConfig({ // --no-isolate = 205s wall vs 48s (module state thrashes across files), // threads = no change. slowTestThreshold: 500, - reporters: ['default', './scripts/vitest-slow-test-reporter.ts'], + reporters: ['default', slowTestGateReporter()], projects: [ { test: {