Refactor audio and video infrastructure in the RunAnywhere Web SDK

- Updated imports in `speak.ts`, `transcribe.ts`, `vision.ts`, and `voice.ts` to use the new core package for audio and video functionalities.
- Introduced new `AudioCapture`, `AudioPlayback`, and `VideoCapture` classes to handle audio and video processing more efficiently.
- Added backend-agnostic types for STT, TTS, LLM, and VLM in the core types module.
- Implemented streaming capabilities for model downloads in `ModelDownloader`.
- Enhanced the overall structure for better modularity and maintainability.

Author: sanchitmonga22

examples/web/RunAnywhereAI/src/views/speak.ts

@@ -21,9 +21,7 @@ const SURPRISE_TEXTS = [
 ];
 
 let ttsIsSpeaking = false;
-let ttsPlayback: InstanceType<
-  typeof import('../../../../../sdk/runanywhere-web/packages/onnx/src/Infrastructure/AudioPlayback').AudioPlayback
-> | null = null;
+let ttsPlayback: import('../../../../../sdk/runanywhere-web/packages/core/src/Infrastructure/AudioPlayback').AudioPlayback | null = null;
 
 // ---------------------------------------------------------------------------
 // Init
@@ -139,7 +137,10 @@ async function handleSpeak(): Promise<void> {
     statusEl.textContent = 'Synthesizing speech...';
     const speed = parseFloat(speedSlider.value);
 
-    const { TTS, AudioPlayback } = await import(
+    const { AudioPlayback } = await import(
+      '../../../../../sdk/runanywhere-web/packages/core/src/index'
+    );
+    const { TTS } = await import(
       '../../../../../sdk/runanywhere-web/packages/onnx/src/index'
     );
 

examples/web/RunAnywhereAI/src/views/transcribe.ts

@@ -4,7 +4,8 @@
  */
 
 import type { TabLifecycle } from '../app';
-import { AudioCapture, VAD, SpeechActivity } from '../../../../../sdk/runanywhere-web/packages/onnx/src/index';
+import { AudioCapture, SpeechActivity } from '../../../../../sdk/runanywhere-web/packages/core/src/index';
+import { VAD } from '../../../../../sdk/runanywhere-web/packages/onnx/src/index';
 import { ModelManager, ModelCategory, ensureVADLoaded, type ModelInfo } from '../services/model-manager';
 import { showModelSelectionSheet } from '../components/model-selection';
 

examples/web/RunAnywhereAI/src/views/vision.ts

@@ -12,7 +12,8 @@
 import type { TabLifecycle } from '../app';
 import { ModelManager, ModelCategory, type ModelInfo } from '../services/model-manager';
 import { showModelSelectionSheet } from '../components/model-selection';
-import { VLMWorkerBridge, VideoCapture, type CapturedFrame } from '../../../../../sdk/runanywhere-web/packages/llamacpp/src/index';
+import { VideoCapture, type CapturedFrame } from '../../../../../sdk/runanywhere-web/packages/core/src/index';
+import { VLMWorkerBridge } from '../../../../../sdk/runanywhere-web/packages/llamacpp/src/index';
 
 // ---------------------------------------------------------------------------
 // Constants (matching iOS VLMViewModel defaults)

examples/web/RunAnywhereAI/src/views/voice.ts

@@ -8,10 +8,8 @@
 import type { TabLifecycle } from '../app';
 import { showModelSelectionSheet } from '../components/model-selection';
 import { ModelManager, ModelCategory, ensureVADLoaded } from '../services/model-manager';
-import { VoicePipeline, PipelineState } from '../../../../../sdk/runanywhere-web/packages/core/src/index';
-import {
-  AudioCapture, AudioPlayback, VAD, SpeechActivity,
-} from '../../../../../sdk/runanywhere-web/packages/onnx/src/index';
+import { VoicePipeline, PipelineState, AudioCapture, AudioPlayback, SpeechActivity } from '../../../../../sdk/runanywhere-web/packages/core/src/index';
+import { VAD } from '../../../../../sdk/runanywhere-web/packages/onnx/src/index';
 
 /** Shared AudioCapture instance for this view (replaces app-level MicCapture singleton). */
 const micCapture = new AudioCapture();

…/onnx/src/Infrastructure/AudioCapture.ts …/core/src/Infrastructure/AudioCapture.tssdk/runanywhere-web/packages/onnx/src/Infrastructure/AudioCapture.ts renamed to sdk/runanywhere-web/packages/core/src/Infrastructure/AudioCapture.ts sdk/runanywhere-web/packages/onnx/src/Infrastructure/AudioCapture.ts renamed to sdk/runanywhere-web/packages/core/src/Infrastructure/AudioCapture.ts

@@ -12,7 +12,7 @@
  *   - Audio level monitoring via AnalyserNode (currentLevel getter)
  */
 
-import { SDKLogger } from '@runanywhere/web';
+import { SDKLogger } from '../Foundation/SDKLogger';
 
 const logger = new SDKLogger('AudioCapture');
 

…nx/src/Infrastructure/AudioFileLoader.ts …re/src/Infrastructure/AudioFileLoader.tssdk/runanywhere-web/packages/onnx/src/Infrastructure/AudioFileLoader.ts renamed to sdk/runanywhere-web/packages/core/src/Infrastructure/AudioFileLoader.ts sdk/runanywhere-web/packages/onnx/src/Infrastructure/AudioFileLoader.ts renamed to sdk/runanywhere-web/packages/core/src/Infrastructure/AudioFileLoader.ts

@@ -12,7 +12,9 @@
  *   - Completion callbacks
  */
 
-import { SDKLogger, EventBus, SDKEventType } from '@runanywhere/web';
+import { SDKLogger } from '../Foundation/SDKLogger';
+import { EventBus } from '../Foundation/EventBus';
+import { SDKEventType } from '../types/enums';
 
 const logger = new SDKLogger('AudioPlayback');
 

…onnx/src/Infrastructure/AudioPlayback.ts …core/src/Infrastructure/AudioPlayback.tssdk/runanywhere-web/packages/onnx/src/Infrastructure/AudioPlayback.ts renamed to sdk/runanywhere-web/packages/core/src/Infrastructure/AudioPlayback.ts sdk/runanywhere-web/packages/onnx/src/Infrastructure/AudioPlayback.ts renamed to sdk/runanywhere-web/packages/core/src/Infrastructure/AudioPlayback.ts

@@ -224,10 +224,11 @@ export class ModelDownloader {
       let totalBytesDownloaded = 0;
       let totalBytesExpected = 0;
 
-      // Download the primary file
-      const primaryData = await this.downloadFile(model.url, (progress, bytesDown, bytesTotal) => {
+      // Try streaming the primary file directly to storage (keeps memory constant).
+      // Falls back to buffered download + store if streaming is not possible.
+      const primaryProgressCb = (progress: number, bytesDown: number, bytesTotal: number) => {
         totalBytesDownloaded = bytesDown;
-        totalBytesExpected = bytesTotal * totalFiles; // rough estimate
+        totalBytesExpected = bytesTotal * totalFiles;
         const overallProgress = progress / totalFiles;
         this.registry.updateModel(modelId, { downloadProgress: overallProgress });
         this.emitDownloadProgress({
@@ -240,16 +241,21 @@ export class ModelDownloader {
           filesCompleted: 0,
           filesTotal: totalFiles,
         });
-      });
+      };
 
-      await this.storeInOPFS(modelId, primaryData);
+      let primarySize = await this.downloadAndStoreStreaming(model.url, modelId, primaryProgressCb);
+      if (primarySize === null) {
+        const primaryData = await this.downloadFile(model.url, primaryProgressCb);
+        await this.storeInOPFS(modelId, primaryData);
+        primarySize = primaryData.length;
+      }
 
       // Download additional files (e.g., mmproj for VLM)
       if (model.additionalFiles && model.additionalFiles.length > 0) {
         for (let i = 0; i < model.additionalFiles.length; i++) {
           const file = model.additionalFiles[i];
           const fileKey = this.additionalFileKey(modelId, file.filename);
-          const fileData = await this.downloadFile(file.url, (progress, bytesDown, bytesTotal) => {
+          const fileProgressCb = (progress: number, bytesDown: number, bytesTotal: number) => {
             const baseProgress = (1 + i) / totalFiles;
             const fileProgress = progress / totalFiles;
             const overallProgress = baseProgress + fileProgress;
@@ -264,8 +270,13 @@ export class ModelDownloader {
               filesCompleted: 1 + i,
               filesTotal: totalFiles,
             });
-          });
-          await this.storeInOPFS(fileKey, fileData);
+          };
+
+          const streamedSize = await this.downloadAndStoreStreaming(file.url, fileKey, fileProgressCb);
+          if (streamedSize === null) {
+            const fileData = await this.downloadFile(file.url, fileProgressCb);
+            await this.storeInOPFS(fileKey, fileData);
+          }
         }
       }
 
@@ -280,7 +291,7 @@ export class ModelDownloader {
         filesTotal: totalFiles,
       });
 
-      let totalSize = primaryData.length;
+      let totalSize = primarySize;
       if (model.additionalFiles) {
         for (const file of model.additionalFiles) {
           const fileKey = this.additionalFileKey(modelId, file.filename);
@@ -359,6 +370,56 @@ export class ModelDownloader {
     return data;
   }
 
+  /**
+   * Download a file and stream it directly to persistent storage (OPFS or local FS)
+   * without buffering the entire payload in memory.
+   *
+   * Returns the total bytes downloaded. Falls back to buffered download + store
+   * if streaming write is not supported or fails.
+   *
+   * @returns Total bytes written, or null if streaming was not possible.
+   */
+  async downloadAndStoreStreaming(
+    url: string,
+    storageKey: string,
+    onProgress?: (progress: number, bytesDownloaded: number, totalBytes: number) => void,
+  ): Promise<number | null> {
+    validateModelUrl(url);
+    const response = await fetch(url);
+    if (!response.ok) throw new Error(`HTTP ${response.status} for ${url}`);
+    if (!response.body) return null;
+
+    const total = Number(response.headers.get('content-length') || 0);
+    let received = 0;
+
+    // Build a progress-tracking pass-through stream
+    const progressTransform = new TransformStream<Uint8Array, Uint8Array>({
+      transform: (chunk, controller) => {
+        received += chunk.length;
+        onProgress?.(total > 0 ? received / total : 0, received, total);
+        controller.enqueue(chunk);
+      },
+    });
+
+    const storageStream = response.body.pipeThrough(progressTransform);
+
+    try {
+      if (this.localFileStorage?.isReady) {
+        await this.localFileStorage.saveModelFromStream(storageKey, storageStream);
+        logger.info(`Streamed ${storageKey} to local storage (${(received / 1024 / 1024).toFixed(1)} MB)`);
+        return received;
+      }
+
+      await this.storage.saveModelFromStream(storageKey, storageStream);
+      logger.info(`Streamed ${storageKey} to OPFS (${(received / 1024 / 1024).toFixed(1)} MB)`);
+      return received;
+    } catch (err) {
+      const msg = err instanceof Error ? err.message : String(err);
+      logger.warning(`Streaming store failed for "${storageKey}": ${msg}, will fall back to buffered download`);
+      return null;
+    }
+  }
+
   /** Store data, preferring local filesystem when available, then OPFS, then memory cache. */
   async storeInOPFS(key: string, data: Uint8Array): Promise<void> {
     const sizeMB = (data.length / 1024 / 1024).toFixed(1);

sdk/runanywhere-web/packages/core/src/Infrastructure/ModelDownloader.ts

@@ -6,10 +6,14 @@
  * Core code (e.g. VoicePipeline) retrieves them at runtime via
  * `ExtensionPoint.getProvider()` with full compile-time type safety.
  *
- * Replaces the previous implicit `globalThis.__runanywhere_*` contract.
- * See: https://github.com/RunanywhereAI/runanywhere-sdks/issues/371
+ * All referenced types (LLMGenerationResult, STTTranscriptionResult, etc.)
+ * are defined in core so providers return properly typed results.
  */
 
+import type { LLMGenerationResult } from '../types/LLMTypes';
+import type { STTTranscriptionResult, STTTranscribeOptions } from '../types/STTTypes';
+import type { TTSSynthesisResult, TTSSynthesizeOptions } from '../types/TTSTypes';
+
 // ---------------------------------------------------------------------------
 // Provider Capability Keys
 // ---------------------------------------------------------------------------
@@ -26,10 +30,6 @@ export type ProviderCapability = 'llm' | 'stt' | 'tts';
 
 /**
  * LLM (text generation) provider — implemented by @runanywhere/web-llamacpp.
- *
- * Only the subset of the TextGeneration API that cross-package consumers
- * (e.g. VoicePipeline) depend on. Backend packages may expose additional
- * methods beyond this interface.
  */
 export interface LLMProvider {
   generateStream(
@@ -41,47 +41,29 @@ export interface LLMProvider {
     },
   ): Promise<{
     stream: AsyncIterable<string>;
-    result: Promise<{
-      text: string;
-      tokensUsed: number;
-      tokensPerSecond: number;
-      [key: string]: unknown;
-    }>;
+    result: Promise<LLMGenerationResult>;
     cancel: () => void;
   }>;
 }
 
 /**
  * STT (speech-to-text) provider — implemented by @runanywhere/web-onnx.
- *
- * Only the subset of the STT API that cross-package consumers depend on.
  */
 export interface STTProvider {
   transcribe(
     audio: Float32Array,
-    options?: { sampleRate?: number },
-  ): Promise<{
-    text: string;
-    [key: string]: unknown;
-  }>;
+    options?: STTTranscribeOptions,
+  ): Promise<STTTranscriptionResult>;
 }
 
 /**
  * TTS (text-to-speech) provider — implemented by @runanywhere/web-onnx.
- *
- * Only the subset of the TTS API that cross-package consumers depend on.
  */
 export interface TTSProvider {
   synthesize(
     text: string,
-    options?: { speed?: number },
-  ): Promise<{
-    audioData: Float32Array;
-    sampleRate: number;
-    durationMs: number;
-    processingTimeMs: number;
-    [key: string]: unknown;
-  }>;
+    options?: TTSSynthesizeOptions,
+  ): Promise<TTSSynthesisResult>;
 }
 
 // ---------------------------------------------------------------------------

sdk/runanywhere-web/packages/core/src/Infrastructure/ProviderTypes.ts

@@ -26,7 +26,7 @@
  *   ```
  */
 
-import { SDKLogger } from '@runanywhere/web';
+import { SDKLogger } from '../Foundation/SDKLogger';
 
 const logger = new SDKLogger('VideoCapture');