[Web-SDK] [Web-Example] Web SDK fixes (#433)

* Refactor audio and video infrastructure in the RunAnywhere Web SDK

- Updated imports in `speak.ts`, `transcribe.ts`, `vision.ts`, and `voice.ts` to use the new core package for audio and video functionalities.
- Introduced new `AudioCapture`, `AudioPlayback`, and `VideoCapture` classes to handle audio and video processing more efficiently.
- Added backend-agnostic types for STT, TTS, LLM, and VLM in the core types module.
- Implemented streaming capabilities for model downloads in `ModelDownloader`.
- Enhanced the overall structure for better modularity and maintainability.

* minor fixes

* Upgrade llama.cpp to b8179 and fix WASM VLM build for new mtmd model sources

- Bump LLAMACPP_VERSION from b8011 to b8179 for Vulkan/WebGPU precision
  fixes and FlashAttention improvements
- Adapt LoRA adapter API calls in llamacpp_backend.cpp to the new
  llama_set_adapters_lora() batch API (replaces removed per-adapter functions)
- Add nemotron-v2-vl.cpp and paddleocr.cpp to CMakeLists.txt for new
  clip_graph_* model types introduced in b8179
- Document Qwen2-VL CPU fallback performance (~1 tok/s) caused by
  M-RoPE NaN logits on WebGPU in rac_vlm_llamacpp.cpp,
  VLMWorkerBridge.ts, and model-manager.ts

* addressed code rabbit comments

* minor fixes

* bumped up the version

* Refactor VLM model type resolution and system prompt handling

- Updated the logic for determining the effective system prompt to handle empty strings.
- Introduced a new function to resolve the effective VLM model type based on options, simplifying the code in `rac_vlm_llamacpp_process`.
- Improved download progress tracking in `ModelDownloader` by using cumulative byte counts for better accuracy.
- Enhanced type exports in `llamacpp` and `onnx` packages for better compatibility and clarity.
- Adjusted regex for identifying Qwen VL models to ensure more accurate matching.

---------

Co-authored-by: Shubham Malhotra <shubham.malhotra28@gmail.com>

Author: sanchitmonga22

examples/web/RunAnywhereAI/src/services/model-manager.ts

@@ -102,6 +102,8 @@ const REGISTERED_MODELS: CompactModelDef[] = [
     modality: ModelCategory.Multimodal,
     memoryRequirement: 600_000_000,
   },
+  // NOTE: Qwen2-VL uses M-RoPE which produces NaN logits on WebGPU. It falls
+  // back to CPU WASM (~1 tok/s) — noticeably slower than LFM2-VL on WebGPU.
   {
     id: 'qwen2-vl-2b-instruct-q4_k_m',
     name: 'Qwen2-VL 2B Instruct Q4_K_M',

examples/web/RunAnywhereAI/src/views/speak.ts

@@ -21,9 +21,7 @@ const SURPRISE_TEXTS = [
 ];
 
 let ttsIsSpeaking = false;
-let ttsPlayback: InstanceType<
-  typeof import('../../../../../sdk/runanywhere-web/packages/onnx/src/Infrastructure/AudioPlayback').AudioPlayback
-> | null = null;
+let ttsPlayback: import('../../../../../sdk/runanywhere-web/packages/core/src/Infrastructure/AudioPlayback').AudioPlayback | null = null;
 
 // ---------------------------------------------------------------------------
 // Init
@@ -139,7 +137,10 @@ async function handleSpeak(): Promise<void> {
     statusEl.textContent = 'Synthesizing speech...';
     const speed = parseFloat(speedSlider.value);
 
-    const { TTS, AudioPlayback } = await import(
+    const { AudioPlayback } = await import(
+      '../../../../../sdk/runanywhere-web/packages/core/src/index'
+    );
+    const { TTS } = await import(
       '../../../../../sdk/runanywhere-web/packages/onnx/src/index'
     );
 

examples/web/RunAnywhereAI/src/views/transcribe.ts

@@ -4,7 +4,8 @@
  */
 
 import type { TabLifecycle } from '../app';
-import { AudioCapture, VAD, SpeechActivity } from '../../../../../sdk/runanywhere-web/packages/onnx/src/index';
+import { AudioCapture, SpeechActivity } from '../../../../../sdk/runanywhere-web/packages/core/src/index';
+import { VAD } from '../../../../../sdk/runanywhere-web/packages/onnx/src/index';
 import { ModelManager, ModelCategory, ensureVADLoaded, type ModelInfo } from '../services/model-manager';
 import { showModelSelectionSheet } from '../components/model-selection';
 

examples/web/RunAnywhereAI/src/views/vision.ts

@@ -12,7 +12,8 @@
 import type { TabLifecycle } from '../app';
 import { ModelManager, ModelCategory, type ModelInfo } from '../services/model-manager';
 import { showModelSelectionSheet } from '../components/model-selection';
-import { VLMWorkerBridge, VideoCapture, type CapturedFrame } from '../../../../../sdk/runanywhere-web/packages/llamacpp/src/index';
+import { VideoCapture, type CapturedFrame } from '../../../../../sdk/runanywhere-web/packages/core/src/index';
+import { VLMWorkerBridge } from '../../../../../sdk/runanywhere-web/packages/llamacpp/src/index';
 
 // ---------------------------------------------------------------------------
 // Constants (matching iOS VLMViewModel defaults)
@@ -412,7 +413,7 @@ async function processFrame(frame: CapturedFrame, prompt: string, maxTokens: num
       frame.width,
       frame.height,
       prompt,
-      { maxTokens, temperature: 0.7 },
+      { maxTokens, temperature: 0.7, systemPrompt: 'You are a helpful assistant.' },
     );
 
     // Compute metrics from JS wall clock

examples/web/RunAnywhereAI/src/views/voice.ts

@@ -8,10 +8,8 @@
 import type { TabLifecycle } from '../app';
 import { showModelSelectionSheet } from '../components/model-selection';
 import { ModelManager, ModelCategory, ensureVADLoaded } from '../services/model-manager';
-import { VoicePipeline, PipelineState } from '../../../../../sdk/runanywhere-web/packages/core/src/index';
-import {
-  AudioCapture, AudioPlayback, VAD, SpeechActivity,
-} from '../../../../../sdk/runanywhere-web/packages/onnx/src/index';
+import { VoicePipeline, PipelineState, AudioCapture, AudioPlayback, SpeechActivity } from '../../../../../sdk/runanywhere-web/packages/core/src/index';
+import { VAD } from '../../../../../sdk/runanywhere-web/packages/onnx/src/index';
 
 /** Shared AudioCapture instance for this view (replaces app-level MicCapture singleton). */
 const micCapture = new AudioCapture();

sdk/runanywhere-commons/src/backends/llamacpp/llamacpp_backend.cpp

@@ -370,6 +370,7 @@ bool LlamaCppTextGeneration::unload_model_internal() {
 
     // Clear LoRA adapters from context before freeing
     // (adapter memory is freed automatically with the model per llama.cpp API)
+    // Best-effort during teardown: log but don't fail unload on error.
     if (context_ && !lora_adapters_.empty()) {
         llama_set_adapters_lora(context_, nullptr, 0, nullptr);
     }
@@ -829,7 +830,6 @@ bool LlamaCppTextGeneration::recreate_context() {
 
 bool LlamaCppTextGeneration::apply_lora_adapters() {
     if (lora_adapters_.empty()) {
-        // Clear all adapters from context
         llama_set_adapters_lora(context_, nullptr, 0, nullptr);
         return true;
     }
@@ -930,8 +930,6 @@ bool LlamaCppTextGeneration::remove_lora_adapter(const std::string& adapter_path
         return false;
     }
 
-    // Remove from tracking (adapter memory is freed automatically with the model
-    // per llama.cpp API — llama_adapter_lora_free is deprecated since b8011)
     lora_adapters_.erase(it);
 
     // Re-apply remaining adapters (or clear if none left)