diff --git a/__tests__/watcher.test.ts b/__tests__/watcher.test.ts index b372fc3d6..ee8e6179e 100644 --- a/__tests__/watcher.test.ts +++ b/__tests__/watcher.test.ts @@ -18,6 +18,7 @@ */ import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest'; +import type { EventEmitter } from 'events'; import * as fs from 'fs'; import * as path from 'path'; import * as os from 'os'; @@ -25,6 +26,7 @@ import { FileWatcher, LockUnavailableError, __emitWatchEventForTests, + __setFsWatchForTests, type WatchOptions, } from '../src/sync/watcher'; import CodeGraph from '../src/index'; @@ -69,6 +71,8 @@ describe('FileWatcher', () => { }); afterEach(() => { + __setFsWatchForTests(null); + vi.restoreAllMocks(); if (fs.existsSync(testDir)) { fs.rmSync(testDir, { recursive: true, force: true }); } @@ -87,6 +91,63 @@ describe('FileWatcher', () => { expect(watcher.isActive()).toBe(false); }); + it('should not start when fs.watch setup exhausts watch/file resources', () => { + const syncFn = vi.fn().mockResolvedValue({ filesChanged: 0, durationMs: 0 }); + const onDegraded = vi.fn(); + const watcher = new FileWatcher(testDir, syncFn, { debounceMs: 100, onDegraded }); + const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => {}); + __setFsWatchForTests(() => { + const err = new Error('too many open files') as NodeJS.ErrnoException; + err.code = 'EMFILE'; + throw err; + }); + + expect(watcher.start()).toBe(false); + expect(watcher.isActive()).toBe(false); + expect(onDegraded).toHaveBeenCalledTimes(1); + expect(onDegraded).toHaveBeenCalledWith(expect.stringContaining('auto-sync disabled')); + expect(warnSpy).toHaveBeenCalledWith( + expect.stringContaining('File watcher disabled'), + expect.objectContaining({ + reason: expect.stringContaining('auto-sync disabled'), + }) + ); + }); + + it('should degrade once when the recursive watcher emits EMFILE at runtime', async () => { + const syncFn = vi.fn().mockResolvedValue({ filesChanged: 0, durationMs: 0 }); + const onDegraded = vi.fn(); + const handlers = new Map void>>(); + const fakeWatcher = { + on: vi.fn((event: string, handler: (arg?: unknown) => void) => { + const list = handlers.get(event) ?? []; + list.push(handler); + handlers.set(event, list); + return fakeWatcher; + }), + close: vi.fn(), + } as unknown as fs.FSWatcher & EventEmitter; + __setFsWatchForTests(() => fakeWatcher); + const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => {}); + const watcher = new FileWatcher(testDir, syncFn, { debounceMs: 100, onDegraded }); + + expect(watcher.start()).toBe(true); + expect(watcher.isActive()).toBe(true); + + const err = new Error('too many open files') as NodeJS.ErrnoException; + err.code = 'EMFILE'; + for (const handler of handlers.get('error') ?? []) handler(err); + for (const handler of handlers.get('error') ?? []) handler(err); + + expect(watcher.isActive()).toBe(false); + expect(onDegraded).toHaveBeenCalledTimes(1); + expect(fakeWatcher.close).toHaveBeenCalledTimes(1); + const disableCalls = warnSpy.mock.calls.filter( + (call) => typeof call[0] === 'string' && String(call[0]).includes('File watcher disabled') + ); + expect(disableCalls).toHaveLength(1); + }); + it('should be idempotent on double start', () => { const syncFn = vi.fn().mockResolvedValue({ filesChanged: 0, durationMs: 0 }); const watcher = newWatcher(syncFn); @@ -306,7 +367,7 @@ describe('FileWatcher', () => { expect(onSyncError).not.toHaveBeenCalled(); expect(onSyncComplete).not.toHaveBeenCalled(); - await waitFor(() => syncFn.mock.calls.length >= 2); + await waitFor(() => syncFn.mock.calls.length >= 2, 3000); await waitFor( () => !watcher.getPendingFiles().some((p) => p.path === 'src/locked.ts'), ); @@ -317,6 +378,39 @@ describe('FileWatcher', () => { watcher.stop(); }); + + it('should disable auto-sync after prolonged LockUnavailableError contention', async () => { + const syncFn = vi.fn().mockRejectedValue(new LockUnavailableError()); + const onSyncComplete = vi.fn(); + const onSyncError = vi.fn(); + const onDegraded = vi.fn(); + const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => {}); + const watcher = newWatcher(syncFn, { + debounceMs: 25, + onSyncComplete, + onSyncError, + onDegraded, + }); + watcher.start(); + await watcher.waitUntilReady(); + + __emitWatchEventForTests(testDir, 'src/long-lock.ts'); + + await waitFor(() => !watcher.isActive(), 8000, 20); + + expect(syncFn.mock.calls.length).toBeGreaterThanOrEqual(6); + expect(watcher.getPendingFiles()).toEqual([]); + expect(onSyncComplete).not.toHaveBeenCalled(); + expect(onSyncError).not.toHaveBeenCalled(); + expect(onDegraded).toHaveBeenCalledTimes(1); + expect(onDegraded).toHaveBeenCalledWith(expect.stringContaining('auto-sync disabled')); + const disableCalls = warnSpy.mock.calls.filter( + (call) => typeof call[0] === 'string' && String(call[0]).includes('File watcher disabled') + ); + expect(disableCalls).toHaveLength(1); + + warnSpy.mockRestore(); + }); }); describe('callbacks', () => { diff --git a/src/sync/watcher.ts b/src/sync/watcher.ts index 401fbd721..b6bc12030 100644 --- a/src/sync/watcher.ts +++ b/src/sync/watcher.ts @@ -39,6 +39,18 @@ import { normalizePath } from '../utils'; import { isCodeGraphDataDir } from '../directory'; import { watchDisabledReason } from './watch-policy'; +const MAX_LOCK_RETRIES = 5; +const MAX_LOCK_RETRY_DELAY_MS = 30000; + +function isWatchResourceExhaustion(err: unknown): boolean { + const e = err as NodeJS.ErrnoException | undefined; + if (e?.code === 'EMFILE' || e?.code === 'ENFILE') return true; + if (!e?.code && e?.message) { + return /EMFILE|ENFILE|too many open files/i.test(e.message); + } + return false; +} + /** * Native recursive `fs.watch` is only reliable on macOS and Windows; on Linux * (and AIX) it throws `ERR_FEATURE_UNAVAILABLE_ON_PLATFORM`. We branch on this @@ -48,6 +60,14 @@ function supportsRecursiveWatch(): boolean { return process.platform === 'darwin' || process.platform === 'win32'; } +type WatchFn = typeof fs.watch; +let watchImpl: WatchFn = fs.watch; + +/** @internal Test-only seam to inject a fake fs.watch implementation. */ +export function __setFsWatchForTests(fn: WatchFn | null): void { + watchImpl = fn ?? fs.watch; +} + /** * Upper bound on simultaneously-watched directories on the Linux per-directory * path. Each is one inotify watch; the kernel's `fs.inotify.max_user_watches` @@ -98,6 +118,11 @@ export interface WatchOptions { */ onSyncError?: (error: Error) => void; + /** + * Callback when live watching degrades permanently and auto-sync is disabled. + */ + onDegraded?: (reason: string) => void; + /** * Test-only. When true, `start()` installs NO OS-level fs.watch — the * watcher is "inert" and only the {@link __emitWatchEventForTests} / @@ -164,6 +189,10 @@ export class FileWatcher { private dirWatchers = new Map(); /** Set once the per-directory watch cap is hit, so we log only once. */ private dirCapWarned = false; + /** One-way marker that live watching has been disabled due to runtime degradation. */ + private degradedReason: string | null = null; + /** Consecutive lock-contention retries for watcher-triggered syncs. */ + private lockRetryCount = 0; /** Test-only inert mode: started, but with no OS watcher installed. */ private inert = false; private debounceTimer: ReturnType | null = null; @@ -211,6 +240,7 @@ export class FileWatcher { private readonly syncFn: () => Promise<{ filesChanged: number; durationMs: number }>; private readonly onSyncComplete?: WatchOptions['onSyncComplete']; private readonly onSyncError?: WatchOptions['onSyncError']; + private readonly onDegraded?: WatchOptions['onDegraded']; private readonly inertForTests: boolean; constructor( @@ -223,6 +253,7 @@ export class FileWatcher { this.debounceMs = options.debounceMs ?? 2000; this.onSyncComplete = options.onSyncComplete; this.onSyncError = options.onSyncError; + this.onDegraded = options.onDegraded; this.inertForTests = options.inertForTests ?? false; } @@ -233,6 +264,8 @@ export class FileWatcher { start(): boolean { if (this.recursiveWatcher || this.dirWatchers.size > 0 || this.inert) return true; // Already watching this.stopped = false; + this.degradedReason = null; + this.lockRetryCount = 0; // Some environments make filesystem watching unusable — most notably // WSL2 /mnt/ drives, where the underlying fs.watch calls block long @@ -275,8 +308,15 @@ export class FileWatcher { return true; } catch (err) { // Watcher setup failed (e.g., permission denied, missing directory). - logWarn('Could not start file watcher', { error: String(err) }); - this.stop(); + if (isWatchResourceExhaustion(err)) { + this.degrade( + 'OS watch/file limit exhausted; auto-sync disabled. Run `codegraph sync` (or install git sync hooks) to refresh the graph after changes.', + { error: String(err) } + ); + } else { + logWarn('Could not start file watcher', { error: String(err) }); + this.stop(); + } return false; } } @@ -287,7 +327,7 @@ export class FileWatcher { * it maps straight to a project-relative path. */ private startRecursive(): void { - this.recursiveWatcher = fs.watch( + this.recursiveWatcher = watchImpl( this.projectRoot, { recursive: true, persistent: true }, (_event, filename) => { @@ -296,6 +336,13 @@ export class FileWatcher { } ); this.recursiveWatcher.on('error', (err: unknown) => { + if (isWatchResourceExhaustion(err)) { + this.degrade( + 'OS watch/file limit exhausted; auto-sync disabled. Run `codegraph sync` (or install git sync hooks) to refresh the graph after changes.', + { error: String(err) } + ); + return; + } logWarn('File watcher error', { error: String(err) }); }); } @@ -319,6 +366,7 @@ export class FileWatcher { * sync owns the baseline). */ private watchTree(dir: string, markExisting: boolean): void { + if (this.stopped || this.degradedReason) return; if (this.dirWatchers.has(dir)) return; if (this.dirWatchers.size >= maxDirWatches()) { if (!this.dirCapWarned) { @@ -332,14 +380,29 @@ export class FileWatcher { let w: fs.FSWatcher; try { - w = fs.watch(dir, { persistent: true }, (_event, filename) => + w = watchImpl(dir, { persistent: true }, (_event, filename) => this.handleDirEvent(dir, filename) ); - } catch { - // ENOENT / EACCES / too-many-open-files — skip this directory quietly. + } catch (err) { + if (isWatchResourceExhaustion(err)) { + this.degrade( + 'OS watch/file limit exhausted; auto-sync disabled. Run `codegraph sync` (or install git sync hooks) to refresh the graph after changes.', + { error: String(err), dir } + ); + } + // ENOENT / EACCES / too-many-open-files on one dir — skip non-fatal cases quietly. return; } - w.on('error', () => this.unwatchDir(dir)); + w.on('error', (err: unknown) => { + if (isWatchResourceExhaustion(err)) { + this.degrade( + 'OS watch/file limit exhausted; auto-sync disabled. Run `codegraph sync` (or install git sync hooks) to refresh the graph after changes.', + { error: String(err), dir } + ); + return; + } + this.unwatchDir(dir); + }); this.dirWatchers.set(dir, w); let entries: fs.Dirent[]; @@ -450,6 +513,15 @@ export class FileWatcher { return this.ignoreMatcher.ignores(rel + '/'); } + /** Disable live watching after a terminal runtime failure. */ + private degrade(reason: string, context: Record = {}): void { + if (this.degradedReason) return; + this.degradedReason = reason; + logWarn('File watcher disabled', { projectRoot: this.projectRoot, reason, ...context }); + this.onDegraded?.(reason); + this.stop(); + } + /** * Stop watching for file changes. */ @@ -478,6 +550,7 @@ export class FileWatcher { } this.dirWatchers.clear(); this.dirCapWarned = false; + this.lockRetryCount = 0; this.inert = false; this.pendingFiles.clear(); @@ -528,7 +601,7 @@ export class FileWatcher { } /** - * Schedule a debounced sync. + * Schedule a normal debounced sync after a source edit. */ private scheduleSync(): void { if (this.debounceTimer) { @@ -540,6 +613,19 @@ export class FileWatcher { }, this.debounceMs); } + /** + * Schedule a retry sync after a recoverable failure such as lock contention. + */ + private scheduleRetrySync(delayMs: number): void { + if (this.debounceTimer) { + clearTimeout(this.debounceTimer); + } + this.debounceTimer = setTimeout(() => { + this.debounceTimer = null; + this.flush(); + }, delayMs); + } + /** * Flush pending changes by running sync. * @@ -561,6 +647,7 @@ export class FileWatcher { try { const result = await this.syncFn(); + this.lockRetryCount = 0; // Remove entries whose most recent event predates this sync — those // edits are now in the DB. Entries with lastSeenMs > syncStartedMs // arrived mid-sync; whether the in-flight sync captured them depends @@ -576,13 +663,22 @@ export class FileWatcher { this.onSyncComplete?.(result); } catch (err) { if (err instanceof LockUnavailableError) { + this.lockRetryCount += 1; // Lock-failure no-op (another writer holds the lock). pendingFiles - // stays intact and the `finally` block reschedules. Debug-only — - // a long external index would otherwise spam stderr every cycle. + // stays intact. Keep short contention quiet, but stop retrying forever + // at the normal debounce cadence when another writer is long-lived. logDebug('Watch sync skipped: file lock unavailable', { pendingFiles: this.pendingFiles.size, + retryCount: this.lockRetryCount, }); + if (this.lockRetryCount > MAX_LOCK_RETRIES) { + this.degrade( + 'File lock unavailable for too long; auto-sync disabled. Run `codegraph sync` after the writer finishes (or install git sync hooks) to refresh the graph.', + { pendingFiles: this.pendingFiles.size, retryCount: this.lockRetryCount } + ); + } } else { + this.lockRetryCount = 0; const error = err instanceof Error ? err : new Error(String(err)); logWarn('Watch sync failed', { error: error.message }); this.onSyncError?.(error); @@ -595,7 +691,15 @@ export class FileWatcher { // If pending files remain (mid-sync events, or this sync failed), // schedule another pass. if (this.pendingFiles.size > 0 && !this.stopped) { - this.scheduleSync(); + if (this.lockRetryCount > 0) { + const retryDelayMs = Math.min( + this.debounceMs * (2 ** Math.max(0, this.lockRetryCount - 1)), + MAX_LOCK_RETRY_DELAY_MS, + ); + this.scheduleRetrySync(retryDelayMs); + } else { + this.scheduleSync(); + } } } }