Fix cross-session system Python cache misses on remotes

docs/startup-flow.md

@@ -46,11 +46,12 @@ python environments extension begins activation
 - **If P3 won:** environment is already resolved → done, no `get()` call needed.
 - **Otherwise:** calls `manager.get(scope)`, which has two internal paths:
 
-  **Fast path** (`tryFastPathGet` in `fastPath.ts`) — entered when `_initialized` hasn't completed and scope is a `Uri`:
+  **Fast path** (`tryFastPathGet` in `fastPath.ts`) — entered when `_initialized` hasn't completed and the manager has a persisted path for the requested scope:
   1. Synchronously create `_initialized` deferred + kick off `startBackgroundInit()` (fire-and-forget full PET discovery)
-  2. Read persisted env path from workspace state
-  3. If persisted path exists → `resolve(path)` → return immediately (background init continues in parallel)
-  4. If no persisted path or resolve fails → fall through to slow path
+  2. Read the workspace-scoped path, or for the global system Python read the workspace-state mirror first and global state as a compatibility/recovery fallback
+  3. If the persisted path exists → `resolve(path)` → return immediately (background init continues in parallel)
+  4. If no persisted path or resolve fails → fall through to slow path; definitively missing global paths are invalidated in both stores
+  - Global mirror writes carry a revision and tombstone so partial Memento failures cannot resurrect or overwrite a newer selection
   - *On background init failure:* clears `_initialized` so next `get()` retries
 
   **Slow path** — fast path skipped or failed:

src/common/persistentState.ts

@@ -10,17 +10,20 @@ export interface PersistentState {
 
 class PersistentStateImpl implements PersistentState {
     private clearing: Deferred<void>;
+
     constructor(private readonly momento: Memento) {
         this.clearing = createDeferred<void>();
         this.clearing.resolve();
     }
+
     async get<T>(key: string, defaultValue?: T): Promise<T | undefined> {
         await this.clearing.promise;
         if (defaultValue === undefined) {
             return this.momento.get<T>(key);
         }
         return this.momento.get<T>(key, defaultValue);
     }
+
     async set<T>(key: string, value: T): Promise<void> {
         await this.clearing.promise;
         await this.momento.update(key, value);
@@ -32,12 +35,22 @@ class PersistentStateImpl implements PersistentState {
             traceError('Error while updating state for key:', key);
         }
     }
+
     async clear(keys?: string[]): Promise<void> {
         if (this.clearing.completed) {
-            this.clearing = createDeferred<void>();
-            const _keys = keys ?? this.momento.keys();
-            await Promise.all(_keys.map((key) => this.momento.update(key, undefined)));
-            this.clearing.resolve();
+            const clearing = (this.clearing = createDeferred<void>());
+            try {
+                const _keys = keys ?? this.momento.keys();
+                const results = await Promise.allSettled(
+                    _keys.map((key) => this.momento.update(key, undefined)),
+                );
+                const failure = results.find((result) => result.status === 'rejected');
+                if (failure?.status === 'rejected') {
+                    throw failure.reason;
+                }
+            } finally {
+                clearing.resolve();
+            }
         }
         return this.clearing.promise;
     }
@@ -46,9 +59,14 @@ class PersistentStateImpl implements PersistentState {
 const _workspace = createDeferred<PersistentState>();
 const _global = createDeferred<PersistentState>();
 
+/** Creates a persistent-state wrapper for the provided VS Code Memento. */
+export function createPersistentState(memento: Memento): PersistentState {
+    return new PersistentStateImpl(memento);
+}
+
 export function setPersistentState(context: ExtensionContext): void {
-    _workspace.resolve(new PersistentStateImpl(context.workspaceState));
-    _global.resolve(new PersistentStateImpl(context.globalState));
+    _workspace.resolve(createPersistentState(context.workspaceState));
+    _global.resolve(createPersistentState(context.globalState));
 }
 
 export function getWorkspacePersistentState(): Promise<PersistentState> {

src/managers/builtin/cache.ts

@@ -1,14 +1,126 @@
+import { randomUUID } from 'crypto';
 import { ENVS_EXTENSION_ID } from '../../common/constants';
-import { getGlobalPersistentState, getWorkspacePersistentState } from '../../common/persistentState';
+import { traceError } from '../../common/logging';
+import {
+    getGlobalPersistentState,
+    getWorkspacePersistentState,
+    type PersistentState,
+} from '../../common/persistentState';
 
 export const SYSTEM_WORKSPACE_KEY = `${ENVS_EXTENSION_ID}:system:WORKSPACE_SELECTED`;
 export const SYSTEM_GLOBAL_KEY = `${ENVS_EXTENSION_ID}:system:GLOBAL_SELECTED`;
+export const SYSTEM_GLOBAL_REVISION_KEY = `${SYSTEM_GLOBAL_KEY}:REVISION`;
 
-export async function clearSystemEnvCache(): Promise<void> {
-    const workspaceState = await getWorkspacePersistentState();
-    await workspaceState.clear([SYSTEM_WORKSPACE_KEY]);
-    const globalState = await getGlobalPersistentState();
-    await globalState.clear([SYSTEM_GLOBAL_KEY]);
+type CacheLayer = 'globalState' | 'workspaceState';
+type CacheOperationResult = { succeeded: true } | { succeeded: false; error: unknown };
+
+interface GlobalSelectionCacheEntry {
+    readonly path?: string;
+    readonly revision: string;
+}
+
+interface WorkspaceGlobalSelection {
+    readonly selection?: unknown;
+    readonly revision?: string;
+    readonly failed: boolean;
+}
+
+let globalCacheMutation: Promise<void> = Promise.resolve();
+
+function enqueueGlobalCacheMutation(operation: () => Promise<void>): Promise<void> {
+    const result = globalCacheMutation.then(operation, operation);
+    globalCacheMutation = result.then(
+        () => undefined,
+        () => undefined,
+    );
+    return result;
+}
+
+function isGlobalSelectionCacheEntry(value: unknown): value is GlobalSelectionCacheEntry {
+    if (!value || typeof value !== 'object') {
+        return false;
+    }
+    const entry = value as { path?: unknown; revision?: unknown };
+    return (
+        typeof entry.revision === 'string' &&
+        entry.revision.length > 0 &&
+        (entry.path === undefined || typeof entry.path === 'string')
+    );
+}
+
+function getStoredGlobalSelectionPath(value: unknown): string | undefined {
+    if (typeof value === 'string') {
+        return value || undefined;
+    }
+    return isGlobalSelectionCacheEntry(value) ? value.path : undefined;
+}
+
+function isCurrentWorkspaceSelection(selection: unknown, revision: string | undefined): boolean {
+    if (typeof selection === 'string') {
+        return selection.length > 0 && revision === undefined;
+    }
+    return isGlobalSelectionCacheEntry(selection) && (revision === undefined || selection.revision === revision);
+}
+
+async function readWorkspaceGlobalSelection(): Promise<WorkspaceGlobalSelection> {
+    try {
+        const state = await getWorkspacePersistentState();
+        const [selection, revision] = await Promise.all([
+            state.get<unknown>(SYSTEM_GLOBAL_KEY),
+            state.get<string>(SYSTEM_GLOBAL_REVISION_KEY),
+        ]);
+        return { selection, revision, failed: false };
+    } catch (error) {
+        traceError('Failed to read workspaceState system Python cache:', error);
+        return { failed: true };
+    }
+}
+
+async function readGlobalSelection(): Promise<unknown> {
+    try {
+        const state = await getGlobalPersistentState();
+        return await state.get<unknown>(SYSTEM_GLOBAL_KEY);
+    } catch (error) {
+        traceError('Failed to read globalState system Python cache:', error);
+        return undefined;
+    }
+}
+
+async function runCacheOperation(
+    operation: 'clear' | 'invalidate' | 'update',
+    layer: CacheLayer,
+    action: () => Promise<void>,
+): Promise<CacheOperationResult> {
+    try {
+        await action();
+        return { succeeded: true };
+    } catch (error) {
+        traceError(`Failed to ${operation} ${layer} system Python cache:`, error);
+        return { succeeded: false, error };
+    }
+}
+
+/**
+ * Clears system Python selections from workspace and global persistent state.
+ * Both layers are attempted before the first failure is rethrown.
+ */
+export function clearSystemEnvCache(): Promise<void> {
+    return enqueueGlobalCacheMutation(async () => {
+        const results = await Promise.all([
+            runCacheOperation('clear', 'workspaceState', async () => {
+                const state = await getWorkspacePersistentState();
+                await state.clear([SYSTEM_WORKSPACE_KEY, SYSTEM_GLOBAL_KEY, SYSTEM_GLOBAL_REVISION_KEY]);
+            }),
+            runCacheOperation('clear', 'globalState', async () => {
+                const state = await getGlobalPersistentState();
+                await state.clear([SYSTEM_GLOBAL_KEY]);
+            }),
+        ]);
+        const failure = results.find((result) => !result.succeeded);
+        if (failure && !failure.succeeded) {
+            throw failure.error;
+        }
+    });
 }
 
 export async function getSystemEnvForWorkspace(fsPath: string): Promise<string | undefined> {
@@ -48,12 +160,89 @@ export async function setSystemEnvForWorkspaces(fsPath: string[], envPath: strin
     await state.set(SYSTEM_WORKSPACE_KEY, data);
 }
 
+/**
+ * Returns the persisted global system Python path.
+ *
+ * A complete `workspaceState` entry is primary so the last system Python
+ * resolved for this workspace survives a cold remote `globalState`. The global
+ * entry is read only for legacy cold caches or to recover an incomplete mirrored
+ * write identified by the workspace revision marker.
+ */
 export async function getSystemEnvForGlobal(): Promise<string | undefined> {
-    const state = await getGlobalPersistentState();
-    return await state.get(SYSTEM_GLOBAL_KEY);
+    await globalCacheMutation;
+    const workspace = await readWorkspaceGlobalSelection();
+    if (!workspace.failed && isCurrentWorkspaceSelection(workspace.selection, workspace.revision)) {
+        return getStoredGlobalSelectionPath(workspace.selection);
+    }
+
+    const globalSelection = await readGlobalSelection();
+    if (
+        workspace.revision &&
+        isGlobalSelectionCacheEntry(globalSelection) &&
+        globalSelection.revision === workspace.revision
+    ) {
+        return globalSelection.path;
+    }
+    if (workspace.revision) {
+        return undefined;
+    }
+    return getStoredGlobalSelectionPath(globalSelection);
+}
+
+/**
+ * Removes a stale global system Python path without overwriting a newer selection.
+ * Each layer is compared and updated independently inside the mutation queue.
+ */
+export function invalidateSystemEnvForGlobal(stalePath: string): Promise<void> {
+    return enqueueGlobalCacheMutation(async () => {
+        await Promise.all([
+            runCacheOperation('invalidate', 'workspaceState', async () => {
+                const state = await getWorkspacePersistentState();
+                const selection = await state.get<unknown>(SYSTEM_GLOBAL_KEY);
+                if (getStoredGlobalSelectionPath(selection) === stalePath) {
+                    await state.set(SYSTEM_GLOBAL_KEY, undefined);
+                }
+            }),
+            runCacheOperation('invalidate', 'globalState', async () => {
+                const state = await getGlobalPersistentState();
+                const selection = await state.get<unknown>(SYSTEM_GLOBAL_KEY);
+                if (getStoredGlobalSelectionPath(selection) === stalePath) {
+                    await state.set(SYSTEM_GLOBAL_KEY, undefined);
+                }
+            }),
+        ]);
+    });
 }
 
-export async function setSystemEnvForGlobal(envPath: string | undefined): Promise<void> {
-    const state = await getGlobalPersistentState();
-    await state.set(SYSTEM_GLOBAL_KEY, envPath);
+/**
+ * Persists the global system Python path in both global and workspace state.
+ *
+ * Each write is best-effort and independent so a failure in one storage layer
+ * cannot block discovery or prevent the other layer from being updated. An
+ * `undefined` path is persisted as a tombstone so a failed mirror deletion cannot
+ * resurrect an older fallback value.
+ */
+export function setSystemEnvForGlobal(envPath: string | undefined): Promise<void> {
+    return enqueueGlobalCacheMutation(async () => {
+        const revision = randomUUID();
+        const selection: GlobalSelectionCacheEntry = { path: envPath, revision };
+        let workspaceState: PersistentState | undefined;
+        try {
+            workspaceState = await getWorkspacePersistentState();
+            await workspaceState.set(SYSTEM_GLOBAL_REVISION_KEY, revision);
+        } catch (error) {
+            traceError('Failed to prepare workspaceState system Python cache update:', error);
+        }
+
+        await Promise.all([
+            runCacheOperation('update', 'workspaceState', async () => {
+                const state = workspaceState ?? (await getWorkspacePersistentState());
+                await state.set(SYSTEM_GLOBAL_KEY, selection);
+            }),
+            runCacheOperation('update', 'globalState', async () => {
+                const state = await getGlobalPersistentState();
+                await state.set(SYSTEM_GLOBAL_KEY, selection);
+            }),
+        ]);
+    });
 }

src/managers/builtin/sysPythonManager.ts

@@ -33,6 +33,7 @@ import {
     clearSystemEnvCache,
     getSystemEnvForGlobal,
     getSystemEnvForWorkspace,
+    invalidateSystemEnvForGlobal,
     setSystemEnvForGlobal,
     setSystemEnvForWorkspace,
     setSystemEnvForWorkspaces,
@@ -321,6 +322,7 @@ export class SysPythonManager implements EnvironmentManager {
             getProjectFsPath: (s) => getProjectFsPathForScope(this.api, s),
             getPersistedPath: (fsPath) => getSystemEnvForWorkspace(fsPath),
             getGlobalPersistedPath: () => getSystemEnvForGlobal(),
+            invalidateGlobalPersistedPath: (stalePath) => invalidateSystemEnvForGlobal(stalePath),
             resolve: (p) => resolveSystemPythonEnvironmentPath(p, this.nativeFinder, this.api, this),
             startBackgroundInit: () => this.initializeDiscovery(),
         });
@@ -340,9 +342,7 @@ export class SysPythonManager implements EnvironmentManager {
     async set(scope: SetEnvironmentScope, environment?: PythonEnvironment): Promise<void> {
         if (scope === undefined) {
             this.globalEnv = environment ?? getDefaultGlobalPython(this.collection);
-            if (environment) {
-                await setSystemEnvForGlobal(environment.environmentPath.fsPath);
-            }
+            await setSystemEnvForGlobal(environment?.environmentPath.fsPath);
         }
 
         if (scope instanceof Uri) {

src/managers/common/fastPath.ts

@@ -32,6 +32,8 @@ export interface FastPathOptions {
     startBackgroundInit: () => Promise<void> | Thenable<void>;
     /** Optional: reads the persisted env path for global scope (when scope is undefined). */
     getGlobalPersistedPath?: () => Promise<string | undefined>;
+    /** Optional: removes a definitively stale persisted path for global scope. */
+    invalidateGlobalPersistedPath?: (persistedPath: string) => Promise<void>;
 }
 
 /**
@@ -71,6 +73,17 @@ async function persistedPathExists(persistedPath: string): Promise<boolean> {
     }
 }
 
+async function invalidateGlobalPersistedPath(opts: FastPathOptions, persistedPath: string): Promise<void> {
+    if (!opts.invalidateGlobalPersistedPath) {
+        return;
+    }
+    try {
+        await opts.invalidateGlobalPersistedPath(persistedPath);
+    } catch (error) {
+        traceError(`[${opts.label}] Failed to invalidate stale global path '${persistedPath}':`, error);
+    }
+}
+
 /**
  * Attempts fast-path resolution for manager.get(): if full initialization hasn't completed yet
  * and there's a persisted environment for the workspace, resolve it directly via nativeFinder
@@ -133,6 +146,7 @@ export async function tryFastPathGet(opts: FastPathOptions): Promise<FastPathRes
                 traceVerbose(
                     `[${opts.label}] Fast path: persisted path '${persistedPath}' does not exist, falling through to slow path`,
                 );
+                await invalidateGlobalPersistedPath(opts, persistedPath);
                 return undefined;
             }
             try {