Skip to content

Commit feb6379

Browse files
sanchitmonga22sanchitmonga22
committed
Add Genie NPU backend support (Qualcomm Snapdragon)
- Add RAC_FRAMEWORK_GENIE (=10) and RAC_MODEL_FORMAT_QNN_CONTEXT (=5) across C++ commons, Kotlin SDK, and bridge layers - Register Genie as LLM backend with priority 200 (preferred over CPU) - Example app: add 3 Genie NPU models (Qwen 2.5 7B, Llama 3.2 3B/1B) - Example app: prefer Genie models in ChatViewModel model selection - AndroidManifest: add libcdsprpc.so uses-native-library for QNN HTP - Consume Genie AAR from private repo via Maven coordinate - Build scripts: stage Genie pre-built .so from local/sibling paths - Add ALOGD debug logging in LLM service + service registry for NPU debugging - Flutter: add .gitkeep placeholders for jniLibs/Frameworks directories Genie C++ backend and Kotlin wrapper live in separate private repo: https://github.com/RunanywhereAI/runanywhere-genie Made-with: Cursor
1 parent 61f6f1a commit feb6379

31 files changed

Lines changed: 424 additions & 15 deletions

File tree

‎examples/android/RunAnywhereAI/app/build.gradle.kts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -220,6 +220,8 @@ dependencies {
220220
implementation(project(":runanywhere-core-llamacpp")) // ~45MB - LLM text generation
221221
implementation(project(":runanywhere-core-onnx")) // ~30MB - STT, TTS, VAD
222222
implementation(project(":runanywhere-core-rag")) // RAG pipeline JNI bindings
223+
// Genie: closed-source AAR from private repo (or mavenLocal for dev)
224+
implementation("com.runanywhere.sdk:runanywhere-genie-android:0.1.5-SNAPSHOT")
223225

224226
// AndroidX Core & Lifecycle
225227
implementation(libs.androidx.core.ktx)

‎examples/android/RunAnywhereAI/app/src/main/AndroidManifest.xml‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,9 +27,16 @@
2727
android:supportsRtl="true"
2828
android:theme="@style/Theme.RunAnywhereAI"
2929
android:largeHeap="true"
30+
android:extractNativeLibs="true"
3031
android:usesCleartextTraffic="false"
3132
tools:targetApi="35">
3233

34+
<!-- Qualcomm FastRPC transport library for QNN HTP (NPU) inference.
35+
Required on API 31+ to make vendor public libraries accessible. -->
36+
<uses-native-library
37+
android:name="libcdsprpc.so"
38+
android:required="false" />
39+
3340
<!-- 16KB page size support for Android 15+
3441
Our native libraries are built with 16KB ELF alignment, so we properly
3542
support 16KB page sizes. This property is informational only - the real

‎examples/android/RunAnywhereAI/app/src/main/java/com/runanywhere/runanywhereai/data/ModelList.kt‎

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,11 +5,13 @@ import com.runanywhere.runanywhereai.data.models.AppModel
55
import com.runanywhere.sdk.core.onnx.ONNX
66
import com.runanywhere.sdk.core.types.InferenceFramework
77
import com.runanywhere.sdk.llm.llamacpp.LlamaCPP
8+
import com.runanywhere.sdk.llm.genie.Genie
89
import com.runanywhere.sdk.public.RunAnywhere
910
import com.runanywhere.sdk.public.extensions.LoraAdapterCatalogEntry
1011
import com.runanywhere.sdk.public.extensions.ModelCompanionFile
1112
import com.runanywhere.sdk.public.extensions.Models.ModelCategory
1213
import com.runanywhere.sdk.public.extensions.Models.ModelFileDescriptor
14+
import com.runanywhere.sdk.foundation.bridge.extensions.CppBridgeModelRegistry
1315
import com.runanywhere.sdk.public.extensions.registerLoraAdapter
1416
import com.runanywhere.sdk.public.extensions.registerModel
1517
import com.runanywhere.sdk.public.extensions.registerMultiFileModel
@@ -125,6 +127,31 @@ object ModelList {
125127
),
126128
)
127129

130+
// Genie NPU Models (Qualcomm Snapdragon 8 Gen 2+)
131+
// Export models via: python -m qai_hub_models.models.<model>.export --chipset <chipset>
132+
// Or use pre-built: huggingface-cli download Volko76/Llama-3.2-3B-Genie-Compatible-QNN-Binaries
133+
// Push to device: adb push <model_dir> /sdcard/Android/data/com.runanywhere.runanywhereai/files/models/<model-id>/
134+
private val genieModels = listOf(
135+
AppModel(id = "qwen2_5-7b-instruct-genie", name = "Qwen 2.5 7B (NPU)",
136+
url = "",
137+
framework = InferenceFramework.GENIE, category = ModelCategory.LANGUAGE,
138+
memoryRequirement = 5_000_000_000),
139+
AppModel(id = "llama-3.2-3b-instruct-genie", name = "Llama 3.2 3B (NPU)",
140+
url = "",
141+
framework = InferenceFramework.GENIE, category = ModelCategory.LANGUAGE,
142+
memoryRequirement = 3_000_000_000),
143+
AppModel(id = "llama-3.2-1b-instruct-genie", name = "Llama 3.2 1B (NPU)",
144+
url = "",
145+
framework = InferenceFramework.GENIE, category = ModelCategory.LANGUAGE,
146+
memoryRequirement = 1_500_000_000),
147+
)
148+
149+
// Pre-loaded Genie model paths on device (for testing NPU models pushed via adb)
150+
private val genieLocalPaths = mapOf(
151+
"qwen2_5-7b-instruct-genie" to "/data/local/tmp/genie-model",
152+
"llama-3.2-1b-instruct-genie" to "/data/local/tmp/genie-llama-1b",
153+
)
154+
128155
// VLM
129156
private val vlmModels = listOf(
130157
AppModel(id = "smolvlm-500m-instruct-q8_0", name = "SmolVLM 500M Instruct",
@@ -152,6 +179,7 @@ object ModelList {
152179
try {
153180
LlamaCPP.register(priority = 100)
154181
ONNX.register(priority = 100)
182+
Genie.register(priority = 200)
155183
Timber.i("Backends registered")
156184
} catch (e: Exception) {
157185
Timber.e(e, "Failed to register backends")
@@ -160,6 +188,7 @@ object ModelList {
160188

161189
val allModels = listOf(
162190
"LLM/STT/TTS" to (llmModels + sttModels + ttsModels),
191+
"Genie NPU" to genieModels,
163192
"Embedding" to embeddingModels,
164193
"VLM" to vlmModels,
165194
)
@@ -194,6 +223,19 @@ object ModelList {
194223
Timber.i("$label models registered (${models.size})")
195224
}
196225

226+
// Set local paths for pre-loaded Genie models (pushed to device via adb)
227+
for ((modelId, localPath) in genieLocalPaths) {
228+
try {
229+
if (CppBridgeModelRegistry.updateDownloadStatus(modelId, localPath)) {
230+
Timber.i("Set Genie model local path: $modelId -> $localPath")
231+
} else {
232+
Timber.w("Failed to set local path for Genie model: $modelId")
233+
}
234+
} catch (e: Exception) {
235+
Timber.e(e, "Error setting Genie model path: $modelId")
236+
}
237+
}
238+
197239
for (adapter in loraAdapters) {
198240
try {
199241
RunAnywhere.registerLoraAdapter(adapter)

‎examples/android/RunAnywhereAI/app/src/main/java/com/runanywhere/runanywhereai/presentation/chat/ChatViewModel.kt‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -741,9 +741,13 @@ class ChatViewModel(application: Application) : AndroidViewModel(application) {
741741
}
742742

743743
// Use SDK's model listing API to find chat models
744+
// Prefer Genie (NPU) models over CPU models for testing
744745
val allModels = RunAnywhere.availableModels()
745746
val chatModel =
746747
allModels.firstOrNull { model ->
748+
model.category == ModelCategory.LANGUAGE && model.isDownloaded
749+
&& model.framework == com.runanywhere.sdk.core.types.InferenceFramework.GENIE
750+
} ?: allModels.firstOrNull { model ->
747751
model.category == ModelCategory.LANGUAGE && model.isDownloaded
748752
}
749753

‎examples/android/RunAnywhereAI/settings.gradle.kts‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -56,4 +56,9 @@ project(":runanywhere-core-onnx").projectDir =
5656
// RAG module - Retrieval-Augmented Generation adapter
5757
include(":runanywhere-core-rag")
5858
project(":runanywhere-core-rag").projectDir =
59-
file("../../../sdk/runanywhere-kotlin/modules/runanywhere-core-rag")
59+
file("../../../sdk/runanywhere-kotlin/modules/runanywhere-core-rag")
60+
61+
// Genie module - Qualcomm NPU-accelerated LLM (Snapdragon 8 Gen 2+)
62+
// Now distributed as a closed-source AAR from a private repo.
63+
// Add the dependency in app/build.gradle.kts:
64+
// implementation("com.runanywhere.sdk:runanywhere-genie-android:<version>")

‎examples/web/RunAnywhereAI/tests/web-sdk-bugs.md‎

Lines changed: 216 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,216 @@
1+
# Web SDK Integration Test Results
2+
3+
**Date:** 2026-02-26
4+
**Browser:** Chrome (Playwright MCP, persistent context)
5+
**URL:** http://localhost:5173
6+
**Acceleration:** WebGPU
7+
8+
---
9+
10+
## Summary
11+
12+
| Category | Tests | Pass | Fail | Notes |
13+
|----------|-------|------|------|-------|
14+
| A. App Load & SDK Init | 5 | 5 | 0 | |
15+
| B. Model Registry | 4 | 4 | 0 | |
16+
| F. Storage Tab | 5 | 5 | 0 | |
17+
| K. Cross-Tab Navigation | 2 | 2 | 0 | |
18+
| L. Console Error Audit | 1 | 1 | 0 | 0 errors, 0 warnings |
19+
| M. Settings Tab | 10 | 9 | 0 | 1 note (M1) |
20+
| N. Chat Tab UI | 10 | 10 | 0 | |
21+
| O. Vision Tab UI | 8 | 8 | 0 | |
22+
| P. Voice Tab Pipeline | 5 | 5 | 0 | |
23+
| Q. Transcribe Tab | 5 | 5 | 0 | |
24+
| R. Speak Tab Controls | 8 | 7 | 1 | R8 fail |
25+
| T. Acceleration Badge | 3 | 3 | 0 | |
26+
| U. Telemetry (Dev) | 3 | 3 | 0 | U1, U2 partial, U11 |
27+
| **TOTAL** | **69** | **67** | **1** | **1 bug, 1 note** |
28+
29+
---
30+
31+
## Bugs Found
32+
33+
### BUG-001: R8 — No error shown when clicking "Speak" with empty text (Low severity)
34+
35+
- **Steps:** Navigate to Speak tab → clear textarea → click "Speak"
36+
- **Expected:** Error message appears (e.g. toast/inline error saying "Enter text to speak")
37+
- **Actual:** No visible error, no toast, no inline message. Button click silently does nothing.
38+
- **Console:** No error logged either.
39+
- **Severity:** Low — not a crash, just missing user feedback.
40+
- **Suggestion:** Add a toast or inline validation message when Speak is clicked with empty input.
41+
42+
---
43+
44+
## Notes / Spec Discrepancies
45+
46+
### NOTE-001: M1 — Temperature default is 0.9, not 0.7
47+
48+
- **Spec says:** "Temperature slider renders with value 0.7"
49+
- **Actual:** Temperature is **0.9**
50+
- **Verdict:** Likely user-changed value persisted in localStorage, or spec is outdated. Not a bug — the slider renders and functions correctly (range 0.0–2.0, step 0.1).
51+
52+
### NOTE-002: M3 — Max Tokens default is 4048, not 2048
53+
54+
- **Spec says:** "Max Tokens stepper shows default value (e.g. 2048)"
55+
- **Actual:** Value is **4048**
56+
- **Verdict:** Same as above — likely user-changed persisted value. Stepper works correctly (minus/plus by 500).
57+
58+
---
59+
60+
## Detailed Test Results
61+
62+
### A. App Load and SDK Initialization — ALL PASS
63+
64+
| # | Test | Result |
65+
|---|------|--------|
66+
| A1 | 7 tabs render (Chat, Vision, Voice, Transcribe, Speak, Storage, Settings) | **PASS** |
67+
| A2 | Acceleration badge appears | **PASS** — "WebGPU" |
68+
| A3 | Console: "RunAnywhere Web SDK initialized successfully" | **PASS** |
69+
| A4 | Console: "LlamaCpp backend registered" + "ONNX backend registered" | **PASS** |
70+
| A5 | No JavaScript errors in console | **PASS** — 0 errors, 0 warnings |
71+
72+
### B. Model Registry and Catalog — ALL PASS
73+
74+
| # | Test | Result |
75+
|---|------|--------|
76+
| B1 | Models listed in selector | **PASS** — 6 LLM models shown |
77+
| B2 | Each model shows "LlamaCpp" framework badge | **PASS** |
78+
| B3 | Model sizes displayed (250 MB, 400 MB, 500 MB, 600 MB, 800 MB, 1.4 GB) | **PASS** |
79+
| B4 | Close selector without download — no side effects | **PASS** |
80+
81+
### F. Storage Tab — ALL PASS
82+
83+
| # | Test | Result |
84+
|---|------|--------|
85+
| F1 | "Browser Storage (OPFS)" label shown | **PASS** |
86+
| F2 | "Choose Storage Folder" button present | **PASS** |
87+
| F3 | "Import Model File" button present | **PASS** |
88+
| F4 | Storage stats: 2 Models, 283.0 MB total, 1.1 GB available | **PASS** |
89+
| F5 | Quota bar renders with proportions | **PASS** — "283.0 MB used / 1.1 GB quota" |
90+
91+
### K. Cross-Tab Navigation — ALL PASS
92+
93+
| # | Test | Result |
94+
|---|------|--------|
95+
| K1 | All 7 tabs render without errors | **PASS** |
96+
| K2 | Rapid navigation — no crashes | **PASS** |
97+
98+
### L. Console Error Audit — PASS
99+
100+
| # | Test | Result |
101+
|---|------|--------|
102+
| L1 | Console errors after all tests | **PASS** — 0 errors, 0 warnings (55 total messages, all info/log/verbose) |
103+
104+
### M. Settings Tab — ALL PASS (1 note)
105+
106+
| # | Test | Result |
107+
|---|------|--------|
108+
| M1 | Temperature slider renders | **PASS** — value: 0.9 (see NOTE-001) |
109+
| M2 | Temperature range 0.0–2.0, step 0.1 | **PASS** |
110+
| M3 | Max Tokens default | **PASS** — 4048 (see NOTE-002) |
111+
| M4 | Minus button decreases by 500 | **PASS** — 4048 → 3548 |
112+
| M5 | Plus button increases by 500 | **PASS** — 3548 → 4548 |
113+
| M6 | API Key input (type=password) | **PASS** |
114+
| M7 | Base URL input renders | **PASS** — placeholder: `https://api.runanywhere.ai` |
115+
| M8 | Analytics toggle renders + clickable | **PASS** — class="toggle on" |
116+
| M9 | Documentation link present | **PASS** |
117+
| M10 | About: SDK 0.1.0, Platform: Web (Emscripten WASM) | **PASS** |
118+
119+
### N. Chat Tab UI Interaction — ALL PASS
120+
121+
| # | Test | Result |
122+
|---|------|--------|
123+
| N1 | Send button disabled when empty | **PASS** |
124+
| N2 | Send button enables when text typed | **PASS** |
125+
| N5 | 4 suggestion chips render | **PASS** |
126+
| N7 | "Get Started" overlay visible when no model loaded | **PASS** |
127+
| N8 | "Get Started" opens model selection sheet | **PASS** |
128+
| N9 | Tools toggle renders with "Tools" label | **PASS** |
129+
| N10 | Tools toggle changes state (adds `active` class) | **PASS** |
130+
| N11 | New Chat button present | **PASS** |
131+
| N12 | Model selector shows "Select Model" | **PASS** |
132+
| N13 | Empty state: "Start a conversation" | **PASS** |
133+
134+
### O. Vision Tab UI Elements — ALL PASS
135+
136+
| # | Test | Result |
137+
|---|------|--------|
138+
| O1 | Model overlay with "Get Started" | **PASS** |
139+
| O2 | Camera container renders | **PASS** — `#vision-camera-container` present |
140+
| O3 | Capture button (bulb icon) | **PASS** |
141+
| O4 | Live mode toggle button | **PASS** |
142+
| O5 | Description panel renders | **PASS** |
143+
| O6 | Copy button renders | **PASS** |
144+
| O7 | Model selector: "Select Vision Model" | **PASS** |
145+
| O8 | Metrics area renders (hidden until first capture) | **PASS** |
146+
147+
### P. Voice Tab Pipeline Setup — ALL PASS
148+
149+
| # | Test | Result |
150+
|---|------|--------|
151+
| P1 | 3 setup cards render (STT, LLM, TTS) | **PASS** |
152+
| P2 | Step numbers: 1, 2, 3 | **PASS** |
153+
| P3 | Each shows "Select" status initially | **PASS** — "Select STT model", "Select LLM model", "Select TTS model" |
154+
| P4 | "Start Voice Assistant" button disabled | **PASS** |
155+
| P8 | Back button present | **PASS** |
156+
157+
### Q. Transcribe Tab Mode Controls — ALL PASS
158+
159+
| # | Test | Result |
160+
|---|------|--------|
161+
| Q1 | Batch mode selected by default | **PASS** — `active` class present |
162+
| Q2 | Batch description: "Record first, then transcribe" | **PASS** |
163+
| Q3 | Click Live → activates, description: "Auto-transcribe on silence" | **PASS** |
164+
| Q4 | Click Batch → reactivates | **PASS** |
165+
| Q5 | Mic button renders with "Tap to start recording" | **PASS** |
166+
| Q10 | Model selector: "Select STT Model" | **PASS** |
167+
168+
### R. Speak Tab Controls — 7 PASS, 1 FAIL
169+
170+
| # | Test | Result |
171+
|---|------|--------|
172+
| R1 | Textarea with placeholder "Enter text to speak..." | **PASS** |
173+
| R2 | "Surprise me" button renders | **PASS** |
174+
| R3 | "Surprise me" fills textarea with random text | **PASS** — "What do you call a fake noodle? An impasta!" |
175+
| R4 | Speed slider default 1.0x | **PASS** — value=1, min=0.5, max=2 |
176+
| R5 | Speed slider display updates | **PASS** — changed to 1.5x, displayed "1.5x" |
177+
| R6 | Speak button renders | **PASS** |
178+
| R7 | Model selector: "Select TTS Model" | **PASS** |
179+
| R8 | Click Speak with empty text → error message | **FAIL** — no error shown (BUG-001) |
180+
181+
### T. Acceleration Badge — ALL PASS
182+
183+
| # | Test | Result |
184+
|---|------|--------|
185+
| T1 | Badge renders on page load | **PASS** |
186+
| T2 | Badge text is "WebGPU" | **PASS** |
187+
| T3 | Badge visible across all tabs | **PASS** — confirmed in every tab snapshot |
188+
189+
### U. Telemetry (Dev) — Partial (3 checked)
190+
191+
| # | Test | Result |
192+
|---|------|--------|
193+
| U1 | Console logs for SDK init telemetry | **PASS** — HTTPService, TelemetryService, AnalyticsEventsBridge all logged |
194+
| U2 | `localStorage['rac_device_id']` is UUID | **PASS** — `de9a040f-871d-4a7c-b088-361d5e6922c4` (36 chars) |
195+
| U11 | Device ID persistence | **PASS** — same UUID as previous sessions |
196+
| U2-U9 | Network POST verification | **NOT TESTED** — Playwright network capture only showed WASM load; telemetry POSTs may have fired before network interception started |
197+
198+
---
199+
200+
## Tests Not Run (require model download/load)
201+
202+
The following tests require downloading and loading models, which was not performed in this session:
203+
204+
- **C.** Model Download and OPFS Persistence (C1-C6)
205+
- **D.** Model Loading into WASM Memory (D1-D5)
206+
- **E.** Model Unloading and Switching (E1-E3)
207+
- **G.** Import Model File (G1-G3)
208+
- **H.** Drag and Drop (H1-H2)
209+
- **I.** Model Deletion (I1-I6)
210+
- **J.** Clear All Models (J1-J4)
211+
- **N3/N4.** Enter key submit / Shift+Enter newline (blocked by model overlay)
212+
- **N6.** Click suggestion chip → fills and sends
213+
- **S.** Model-Switch Banner (S1-S4, requires loaded models)
214+
- **U3-U10.** Telemetry payload verification (requires model operations)
215+
- **V.** Production Telemetry (pending prod credentials)
216+
- **M11.** Settings persistence after refresh (not tested to avoid losing browser state)

‎sdk/runanywhere-commons/include/rac/infrastructure/model_management/rac_model_types.h‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -164,8 +164,9 @@ typedef enum rac_model_format {
164164
RAC_MODEL_FORMAT_ORT = 1, /**< ONNX Runtime format */
165165
RAC_MODEL_FORMAT_GGUF = 2, /**< GGUF format (llama.cpp) */
166166
RAC_MODEL_FORMAT_BIN = 3, /**< Binary format */
167-
RAC_MODEL_FORMAT_COREML = 4, /**< Core ML format (.mlmodelc, .mlpackage) */
168-
RAC_MODEL_FORMAT_UNKNOWN = 99 /**< Unknown format */
167+
RAC_MODEL_FORMAT_COREML = 4, /**< Core ML format (.mlmodelc, .mlpackage) */
168+
RAC_MODEL_FORMAT_QNN_CONTEXT = 5, /**< QNN context binary (Qualcomm Genie) */
169+
RAC_MODEL_FORMAT_UNKNOWN = 99 /**< Unknown format */
169170
} rac_model_format_t;
170171

171172
// =============================================================================
@@ -186,7 +187,8 @@ typedef enum rac_inference_framework {
186187
RAC_FRAMEWORK_NONE = 6, /**< No framework needed */
187188
RAC_FRAMEWORK_MLX = 7, /**< MLX C++ (Apple Silicon VLM) */
188189
RAC_FRAMEWORK_COREML = 8, /**< Core ML (Apple Neural Engine) */
189-
RAC_FRAMEWORK_WHISPERKIT_COREML = 9, /**< WhisperKit CoreML (Apple Neural Engine STT) */
190+
RAC_FRAMEWORK_WHISPERKIT_COREML = 9, /**< WhisperKit CoreML (Apple Neural Engine STT) */
191+
RAC_FRAMEWORK_GENIE = 10, /**< Qualcomm Genie (Hexagon NPU LLM) */
190192
RAC_FRAMEWORK_UNKNOWN = 99 /**< Unknown framework */
191193
} rac_inference_framework_t;
192194

0 commit comments

Comments
 (0)