feat(web): implement streaming model import via WORKERFS mounting (Is… (#394)

* feat(web): implement streaming model import via WORKERFS mounting (Issue #372)

# Conflicts:
#	sdk/runanywhere-web/packages/core/src/Infrastructure/ModelDownloader.ts
#	sdk/runanywhere-web/packages/core/src/Infrastructure/ModelLoaderTypes.ts
#	sdk/runanywhere-web/packages/core/src/Infrastructure/ModelManager.ts
#	sdk/runanywhere-web/packages/core/src/Infrastructure/OPFSStorage.ts
#	sdk/runanywhere-web/packages/llamacpp/src/Extensions/RunAnywhere+TextGeneration.ts

* fix(web): address PR #394 feedback (OOM fallback, local file streaming)

# Conflicts:
#	sdk/runanywhere-web/packages/core/src/Infrastructure/LocalFileStorage.ts
#	sdk/runanywhere-web/packages/core/src/Infrastructure/ModelDownloader.ts
#	sdk/runanywhere-web/packages/llamacpp/src/Extensions/RunAnywhere+TextGeneration.ts

* fix(web): add index signatures

* fix(web): address PR #394 bot review comments

Author: abhayrajjais01

sdk/runanywhere-web/packages/core/src/Infrastructure/LocalFileStorage.ts

@@ -349,6 +349,23 @@ export class LocalFileStorage {
     }
   }
 
+  /**
+   * Get the File object for a model without reading into memory.
+   * Enables streaming / mounting for locally stored files.
+   * @param key - Model identifier
+   */
+  async loadModelFile(key: string): Promise<File | null> {
+    if (!this.dirHandle || !this._isReady) return null;
+
+    try {
+      const filename = this.sanitizeFilename(key);
+      const fileHandle = await this.dirHandle.getFileHandle(filename);
+      return await fileHandle.getFile();
+    } catch {
+      return null;
+    }
+  }
+
   /**
    * Check if a model file exists in local storage.
    * @param key - Model identifier

sdk/runanywhere-web/packages/core/src/Infrastructure/ModelDownloader.ts

@@ -555,9 +555,6 @@ export class ModelDownloader {
       return opfsStream;
     }
 
-    // Clean up corrupted 0-byte entries - we can't easily check length on the stream without consuming it,
-    // so we skip the 0-byte check here for now and rely on loadFromOPFS to clean them up.
-
     // Fall back to in-memory cache
     const cached = this.memoryCache.get(key);
     if (cached) {
@@ -574,6 +571,18 @@ export class ModelDownloader {
     return null;
   }
 
+  /** Load file object from storage (local FS or OPFS) without reading into memory. */
+  async loadModelFile(key: string): Promise<File | null> {
+    // Try local filesystem first
+    if (this.localFileStorage?.isReady) {
+      const file = await this.localFileStorage.loadModelFile(key);
+      if (file) return file;
+    }
+
+    // Try OPFS
+    return this.storage.loadModelFile(key);
+  }
+
   /** Check existence in local storage, OPFS, or in-memory cache. */
   async existsInOPFS(key: string): Promise<boolean> {
     if (this.localFileStorage?.isReady) {

sdk/runanywhere-web/packages/core/src/Infrastructure/ModelLoaderTypes.ts

@@ -34,7 +34,7 @@ export interface ModelLoadContext {
    * Primary model file data (read from storage).
    *
    * Note: This is optional. Backend loaders that support streaming
-   * should prefer `dataStream` to avoid large memory allocations.
+   * should prefer `dataStream` or `file` to avoid large memory allocations.
    */
   data?: Uint8Array;
 
@@ -46,6 +46,15 @@ export interface ModelLoadContext {
    */
   dataStream?: ReadableStream<Uint8Array>;
 
+  /**
+   * Primary model file object.
+   *
+   * When available, this allows backends to mount the file directly into
+   * the WASM virtual filesystem (using WORKERFS) without loading it entirely
+   * into JS memory, preventing OOM crashes with large models.
+   */
+  file?: File;
+
   /**
    * Download a file from a URL. Used for on-demand fetching of
    * companion files that aren't in storage yet.

sdk/runanywhere-web/packages/core/src/Infrastructure/ModelManager.ts

@@ -326,9 +326,24 @@ class ModelManagerImpl {
         if (!data) throw new Error('Model not downloaded — please download the model first.');
         await this.loadVADModel(model, data);
       } else {
-        const dataStream = await this.downloader.loadStreamFromOPFS(modelId);
-        if (!dataStream) throw new Error('Model not downloaded — please download the model first.');
-        await this.loadLLMModel(model, modelId, undefined, dataStream);
+        // Try to get the File object directly (WORKERFS path) to avoid loading into memory
+        const file = await this.downloader.loadModelFile(modelId);
+        let dataStream: ReadableStream<Uint8Array> | undefined;
+        let data: Uint8Array | undefined;
+
+        if (!file) {
+          // Try streaming
+          dataStream = await this.downloader.loadStreamFromOPFS(modelId) ?? undefined;
+
+          if (!dataStream) {
+            // Fallback to legacy buffering
+            data = await this.downloader.loadFromOPFS(modelId) ?? undefined;
+          }
+        }
+
+        if (!file && !dataStream && !data) throw new Error('Model not downloaded — please download the model first.');
+
+        await this.loadLLMModel(model, modelId, data, dataStream, file ?? undefined);
       }
 
       this.loadedByCategory.set(category, modelId);
@@ -461,11 +476,12 @@ class ModelManagerImpl {
   /**
    * Build a ModelLoadContext for passing to backend loaders.
    */
-  private buildLoadContext(model: ManagedModel, data?: Uint8Array, dataStream?: ReadableStream<Uint8Array>): ModelLoadContext {
+  private buildLoadContext(model: ManagedModel, data?: Uint8Array, dataStream?: ReadableStream<Uint8Array>, file?: File): ModelLoadContext {
     return {
       model,
       data,
       dataStream,
+      file,
       downloadFile: (url: string) => this.downloader.downloadFile(url),
       loadFile: (fileKey: string) => this.downloader.loadFromOPFS(fileKey),
       storeFile: (fileKey: string, fileData: Uint8Array) => this.downloader.storeInOPFS(fileKey, fileData),
@@ -478,9 +494,9 @@ class ModelManagerImpl {
    * The loader (in @runanywhere/web-llamacpp) handles writing to its own
    * Emscripten FS and calling the C API.
    */
-  private async loadLLMModel(model: ManagedModel, _modelId: string, data?: Uint8Array, dataStream?: ReadableStream<Uint8Array>): Promise<void> {
+  private async loadLLMModel(model: ManagedModel, _modelId: string, data?: Uint8Array, dataStream?: ReadableStream<Uint8Array>, file?: File): Promise<void> {
     if (!this.llmLoader) throw new Error('No LLM loader registered. Register the @runanywhere/web-llamacpp package.');
-    const ctx = this.buildLoadContext(model, data, dataStream);
+    const ctx = this.buildLoadContext(model, data, dataStream, file);
     await this.llmLoader.loadModelFromData(ctx);
     logger.info(`LLM model loaded: ${model.id}`);
   }
@@ -498,9 +514,18 @@ class ModelManagerImpl {
     if (!mmprojFile) {
       // No mmproj — load as text-only LLM
       logger.warning(`No mmproj found, loading as text-only LLM: ${modelId}`);
-      const dataStream = await this.downloader.loadStreamFromOPFS(modelId);
-      if (!dataStream) throw new Error('Model not downloaded.');
-      await this.loadLLMModel(model, modelId, undefined, dataStream);
+
+      const file = await this.downloader.loadModelFile(modelId);
+      let dataStream: ReadableStream<Uint8Array> | undefined;
+      let data: Uint8Array | undefined;
+
+      if (!file) {
+        dataStream = await this.downloader.loadStreamFromOPFS(modelId) ?? undefined;
+        if (!dataStream) data = await this.downloader.loadFromOPFS(modelId) ?? undefined;
+      }
+
+      if (!file && !dataStream && !data) throw new Error('Model not downloaded.');
+      await this.loadLLMModel(model, modelId, data, dataStream, file ?? undefined);
       return;
     }
 

sdk/runanywhere-web/packages/core/src/Infrastructure/OPFSStorage.ts

@@ -207,6 +207,25 @@ export class OPFSStorage {
     }
   }
 
+  /**
+   * Load model file object from OPFS without reading contents into memory.
+   *
+   * @param key - Model identifier or nested path
+   * @returns File object, or null if not found
+   */
+  async loadModelFile(key: string): Promise<File | null> {
+    if (!this.modelsDir) return null;
+
+    try {
+      const dir = await this.resolveParentDir(key, /* create */ false);
+      const filename = this.resolveFilename(key);
+      const fileHandle = await dir.getFileHandle(filename);
+      return await fileHandle.getFile();
+    } catch {
+      return null;
+    }
+  }
+
   /**
    * Check if a model exists in OPFS.
    *

sdk/runanywhere-web/packages/core/src/types/models.ts

@@ -102,6 +102,7 @@ export interface STTResult {
   confidence: number;
   duration: number;
   alternatives: STTAlternative[];
+  [key: string]: unknown;
 }
 
 export interface STTSegment {

sdk/runanywhere-web/packages/llamacpp/src/Extensions/RunAnywhere+TextGeneration.ts

@@ -32,6 +32,7 @@ const logger = new SDKLogger('TextGeneration');
 class TextGenerationImpl {
   readonly extensionName = 'TextGeneration';
   private _llmComponentHandle = 0;
+  private _mountedPath: string | null = null;
 
   /** Ensure the SDK is initialized and return the bridge. */
   private requireBridge(): LlamaCppBridge {
@@ -75,15 +76,43 @@ class TextGenerationImpl {
    */
   async loadModelFromData(ctx: ModelLoadContext): Promise<void> {
     const bridge = this.requireBridge();
-    const modelPath = `/models/${ctx.model.id}.gguf`;
-    if (ctx.dataStream) {
+    let modelPath: string | null = null;
+    let isMounted = false;
+
+    if (this._mountedPath) {
+      try { bridge.unmount(this._mountedPath); } catch { /* ignore */ }
+      this._mountedPath = null;
+    }
+
+    if (ctx.file) {
+      modelPath = bridge.mountFile(ctx.file);
+      if (modelPath) {
+        isMounted = true;
+        this._mountedPath = modelPath;
+      } else {
+        logger.warning('Mounting failed (WORKERFS unavailable?), falling back to reading file as stream');
+        modelPath = `/models/${ctx.model.id}.gguf`;
+        await bridge.writeFileStream(modelPath, ctx.file.stream() as unknown as ReadableStream<Uint8Array>);
+      }
+    } else if (ctx.dataStream) {
+      modelPath = `/models/${ctx.model.id}.gguf`;
       await bridge.writeFileStream(modelPath, ctx.dataStream);
     } else if (ctx.data) {
+      modelPath = `/models/${ctx.model.id}.gguf`;
       bridge.writeFile(modelPath, ctx.data);
     } else {
       throw new Error('No data provided to loadModelFromData');
     }
-    await this.loadModel(modelPath, ctx.model.id, ctx.model.name);
+
+    try {
+      await this.loadModel(modelPath, ctx.model.id, ctx.model.name);
+    } catch (err) {
+      if (isMounted) {
+        bridge.unmount(modelPath);
+        this._mountedPath = null;
+      }
+      throw err;
+    }
   }
 
   /**
@@ -140,16 +169,24 @@ class TextGenerationImpl {
 
     const bridge = this.requireBridge();
 
-    const result = await bridge.callFunction<number | Promise<number>>(
-      'rac_llm_component_unload',
-      'number',
-      ['number'],
-      [this._llmComponentHandle],
-      { async: true },
-    ) as number;
-    bridge.checkResult(result, 'rac_llm_component_unload');
+    try {
+      const result = await bridge.callFunction<number | Promise<number>>(
+        'rac_llm_component_unload',
+        'number',
+        ['number'],
+        [this._llmComponentHandle],
+        { async: true },
+      ) as number;
+      bridge.checkResult(result, 'rac_llm_component_unload');
 
-    logger.info('LLM model unloaded');
+      logger.info('LLM model unloaded');
+    } finally {
+      // Clean up mounted file if applicable (always run cleanup even if unload fails)
+      if (this._mountedPath) {
+        bridge.unmount(this._mountedPath);
+        this._mountedPath = null;
+      }
+    }
   }
 
   /**

sdk/runanywhere-web/packages/llamacpp/src/Foundation/LlamaCppBridge.ts

@@ -160,6 +160,11 @@ export interface LlamaCppModule {
   FS_createPath?: (parent: string, path: string, canRead: boolean, canWrite: boolean) => void;
   FS_createDataFile?: (parent: string, name: string, data: Uint8Array, canRead: boolean, canWrite: boolean, canOwn: boolean) => void;
   FS_unlink?: (path: string) => void;
+  FS_mkdir?: (path: string) => void;
+  FS_rmdir?: (path: string) => void;
+  FS_mount?: (type: any, opts: any, mountpoint: string) => void;
+  FS_unmount?: (mountpoint: string) => void;
+  WORKERFS?: any;
 
   // Generic index access for dynamic function lookups
   [key: string]: unknown;
@@ -171,6 +176,7 @@ export interface LlamaCppModule {
 
 export class LlamaCppBridge {
   private static _instance: LlamaCppBridge | null = null;
+  private static _nextMountId = 0;
   private _module: LlamaCppModule | null = null;
   private _loaded = false;
   private _loading: Promise<void> | null = null;
@@ -438,6 +444,71 @@ export class LlamaCppBridge {
     try { this.module.FS_unlink?.(path); } catch { /* doesn't exist */ }
   }
 
+  /**
+   * Mount a File object into the WASM filesystem (if WORKERFS is available).
+   * Returns the path to the mounted file, or null if mounting failed/unsupported.
+   *
+   * @param file - The browser File object
+   * @returns The absolute path to the file in WASM FS (e.g. /mnt-123/model.gguf) or null
+   */
+  mountFile(file: File): string | null {
+    const m = this.module;
+    if (!m.FS_mount || !m.WORKERFS) return null;
+
+    let createdMountDir = false;
+    let mountDir = '';
+
+    try {
+      // Create a unique mount point directory
+      const mountId = LlamaCppBridge._nextMountId++;
+      mountDir = `/mnt-${mountId}`;
+
+      if (m.FS_mkdir) {
+        m.FS_mkdir(mountDir);
+        createdMountDir = true;
+      }
+
+      // Mount the file. WORKERFS expects { files: [File, ...] } or { files: [{name, data: File}] }
+      // We assume the standard Emscripten WORKERFS behavior where `files` array mounts them by name.
+      m.FS_mount(m.WORKERFS, { files: [file] }, mountDir);
+
+      logger.debug(`Mounted ${file.name} to ${mountDir}`);
+      return `${mountDir}/${file.name}`;
+    } catch (err) {
+      if (createdMountDir && m.FS_rmdir) {
+        try { m.FS_rmdir(mountDir); } catch { logger.warning(`Failed to clean up mount dir ${mountDir}`); }
+      }
+      const msg = err instanceof Error ? err.message : String(err);
+      logger.warning(`Failed to mount file (WORKERFS): ${msg}`);
+      return null;
+    }
+  }
+
+  /**
+   * Unmount a directory (and remove it).
+   * @param mountDir - The directory path (e.g. /mnt-123)
+   */
+  unmount(mountPath: string): void {
+    if (!mountPath.startsWith('/mnt-')) return; // Safety check
+
+    // Strip filename if present
+    const parts = mountPath.split('/');
+    // formatted like ["", "mnt-123", "filename"]
+    let dir = mountPath;
+    if (parts.length >= 3) {
+      dir = `/${parts[1]}`;
+    }
+
+    try {
+      const m = this.module;
+      if (m.FS_unmount) m.FS_unmount(dir);
+      if (m.FS_rmdir) m.FS_rmdir(dir);
+      logger.debug(`Unmounted ${dir}`);
+    } catch {
+      /* ignore cleanup errors */
+    }
+  }
+
   // -----------------------------------------------------------------------
   // WebGPU Detection
   // -----------------------------------------------------------------------