Merge pull request #477 from RunanywhereAI/smonga/merge-queue-integration

Merging the 9-PR integration into main. All SDKs (iOS Swift, Kotlin JVM, Web, C++ commons) compile clean. C++ commons test suite: 35/35 passing. Android, Flutter, and Web example apps compile clean. iOS example app has a known link failure against the committed prebuilt RACommons.xcframework (which predates #471 VAD API + #469 timing API) — a fresh ./scripts/build-swift.sh --setup is required post-merge to regenerate the xcframeworks.

Preserving the individual merge commits for traceability (no squash).

Author: sanchitmonga22

.gitignore

@@ -388,6 +388,12 @@ tools/
 sdk/runanywhere-react-native/packages/rag/ios/.testlocal
 
 
+# Python virtual environments
+.venv*/
+venv*/
+__pycache__/
+*.pyc
+
 # Node
 node_modules/
 /tools/

.idea/vcs.xml

@@ -43,6 +43,15 @@ let useLocalBinaries = false //  Toggle: true for local dev, false for release
 // Updated automatically by CI/CD during releases
 let sdkVersion = "0.19.7"
 
+// MetalRT remote binary availability flag.
+// Set to `false` until a real checksum for RABackendMetalRT-v<sdkVersion>.zip
+// has been published. When `false`, the MetalRT product/targets are only
+// exposed under `useLocalBinaries = true`, so SPM resolution will not fail
+// for external consumers due to a placeholder checksum.
+let metalrtRemoteBinaryAvailable = false
+
+let includeMetalRT = useLocalBinaries || metalrtRemoteBinaryAvailable
+
 let package = Package(
     name: "runanywhere-sdks",
     platforms: [
@@ -81,7 +90,8 @@ let package = Package(
             name: "RunAnywhereWhisperKit",
             targets: ["WhisperKitRuntime"]
         ),
-    ],
+
+    ] + metalRTProducts(),
     dependencies: [
         .package(url: "https://github.com/apple/swift-crypto.git", from: "3.0.0"),
         .package(url: "https://github.com/Alamofire/Alamofire.git", from: "5.9.0"),
@@ -221,9 +231,62 @@ let package = Package(
             path: "sdk/runanywhere-swift/Tests/RunAnywhereTests"
         ),
 
-    ] + binaryTargets()
+    ] + metalRTTargets() + binaryTargets()
 )
 
+// =============================================================================
+// METALRT PRODUCT / TARGET GATING
+// =============================================================================
+// The RABackendMetalRT.xcframework is not yet published to GitHub releases
+// with a real checksum. To avoid SPM resolution failures for external
+// consumers due to a placeholder zero-checksum binary target, the MetalRT
+// product and its dependent targets are only included when:
+//   - `useLocalBinaries == true` (local dev with a checked-out xcframework), or
+//   - `metalrtRemoteBinaryAvailable == true` (once a real checksum is wired in).
+func metalRTProducts() -> [Product] {
+    guard includeMetalRT else { return [] }
+    return [
+        .library(
+            name: "RunAnywhereMetalRT",
+            targets: ["MetalRTRuntime"]
+        ),
+    ]
+}
+
+func metalRTTargets() -> [Target] {
+    guard includeMetalRT else { return [] }
+    return [
+        // MetalRT C Bridge Module - exposes rac_backend_metalrt_register()
+        .target(
+            name: "MetalRTBackend",
+            dependencies: ["RABackendMetalRTBinary"],
+            path: "sdk/runanywhere-swift/Sources/MetalRTRuntime/include",
+            publicHeadersPath: "."
+        ),
+        // MetalRT Runtime Backend (custom Metal GPU kernels)
+        .target(
+            name: "MetalRTRuntime",
+            dependencies: [
+                "RunAnywhere",
+                "MetalRTBackend",
+                "RABackendMetalRTBinary",
+            ],
+            path: "sdk/runanywhere-swift/Sources/MetalRTRuntime",
+            exclude: ["include"],
+            resources: [
+                .copy("Resources/default.metallib"),
+            ],
+            linkerSettings: [
+                .linkedLibrary("c++"),
+                .linkedFramework("Accelerate"),
+                .linkedFramework("Metal"),
+                .linkedFramework("CoreGraphics"),
+                .linkedFramework("ImageIO"),
+            ]
+        ),
+    ]
+}
+
 // =============================================================================
 // BINARY TARGET SELECTION
 // =============================================================================
@@ -251,6 +314,10 @@ func binaryTargets() -> [Target] {
                 name: "RABackendONNXBinary",
                 path: "sdk/runanywhere-swift/Binaries/RABackendONNX.xcframework"
             ),
+            .binaryTarget(
+                name: "RABackendMetalRTBinary",
+                path: "sdk/runanywhere-swift/Binaries/RABackendMetalRT.xcframework"
+            ),
         ]
 
         // ONNX Runtime xcframeworks - split by platform
@@ -302,6 +369,19 @@ func binaryTargets() -> [Target] {
             ),
         ]
 
+        // MetalRT remote binary is only appended once a real checksum has been
+        // published. Until then the MetalRT product/targets are omitted from
+        // the package graph entirely (see metalRTProducts/metalRTTargets).
+        if metalrtRemoteBinaryAvailable {
+            targets.append(
+                .binaryTarget(
+                    name: "RABackendMetalRTBinary",
+                    url: "https://github.com/RunanywhereAI/runanywhere-sdks/releases/download/v\(sdkVersion)/RABackendMetalRT-v\(sdkVersion).zip",
+                    checksum: "0000000000000000000000000000000000000000000000000000000000000000" // TODO: replace with real checksum
+                )
+            )
+        }
+
         return targets
     }
 }

Package.swift

@@ -41,6 +41,13 @@ struct PlaygroundView: View {
             }
         }
         .task { await viewModel.checkModelStatus() }
+        .onDisappear {
+            // Release audio resources when leaving the tab to prevent conflicts
+            // with FlowSessionManager's AudioCaptureManager.
+            if viewModel.isRecording {
+                Task { await viewModel.toggleRecording() }
+            }
+        }
         .onReceive(NotificationCenter.default.publisher(for: UIApplication.willEnterForegroundNotification)) { _ in
             Task { await viewModel.checkModelStatus() }
         }
@@ -126,18 +133,38 @@ struct PlaygroundView: View {
                 if viewModel.isRecording {
                     // Recording indicator
                     VStack(spacing: 12) {
-                        // Elapsed time
-                        Text(formatTime(viewModel.elapsedSeconds))
-                            .font(.system(size: 20, weight: .semibold, design: .monospaced))
-                            .foregroundStyle(Color.red)
+                        // Elapsed time + speech indicator
+                        HStack(spacing: 8) {
+                            Text(formatTime(viewModel.elapsedSeconds))
+                                .font(.system(size: 20, weight: .semibold, design: .monospaced))
+                                .foregroundStyle(Color.red)
+
+                            if viewModel.isAutoStopEnabled {
+                                Circle()
+                                    .fill(viewModel.speechDetected ? Color.green : Color.gray.opacity(0.4))
+                                    .frame(width: 8, height: 8)
+                            }
+                        }
 
                         // Waveform bars
                         WaveformBars(level: viewModel.audioLevel)
                     }
                 } else {
-                    Text(viewModel.transcription.isEmpty ? "Tap to record" : "Tap to record again")
-                        .font(.subheadline)
-                        .foregroundStyle(AppColors.textTertiary)
+                    VStack(spacing: 12) {
+                        Text(viewModel.transcription.isEmpty ? "Tap to record" : "Tap to record again")
+                            .font(.subheadline)
+                            .foregroundStyle(AppColors.textTertiary)
+
+                        // Auto-stop toggle
+                        Toggle(isOn: $viewModel.isAutoStopEnabled) {
+                            Label("Auto-stop on silence", systemImage: "waveform.badge.minus")
+                                .font(.caption)
+                                .foregroundStyle(AppColors.textSecondary)
+                        }
+                        .toggleStyle(.switch)
+                        .tint(AppColors.primaryGreen)
+                        .frame(width: 240)
+                    }
                 }
             }
         }

Playground/YapRun/YapRun/Features/Playground/PlaygroundView.swift

@@ -25,11 +25,21 @@ final class PlaygroundViewModel {
     var errorMessage: String?
     var modelName: String?
 
+    // MARK: - Auto-Stop (VAD)
+
+    var isAutoStopEnabled = false
+    var speechDetected = false
+
     // MARK: - Private
 
     private let audioCapture = AudioCaptureManager()
     private var audioBuffer = Foundation.Data()
     private var timerTask: Task<Void, Never>?
+    private var vadMonitorTask: Task<Void, Never>?
+    private var vadProcessedBytes = 0
+    private var silenceStartTime: Date?
+    private var hasSpeechBeenDetected = false
+    private let autoStopSilenceDuration: TimeInterval = 2.0
     private let logger = Logger(subsystem: "com.runanywhere.yaprun", category: "Playground")
 
     // MARK: - Model Check
@@ -58,6 +68,15 @@ final class PlaygroundViewModel {
             return
         }
 
+        // Prevent conflict with active voice keyboard session (iOS-only).
+        // FlowSessionManager is compiled `#if os(iOS)` so this check is skipped on macOS.
+        #if os(iOS)
+        guard !FlowSessionManager.shared.isActive else {
+            errorMessage = "Voice keyboard session is active. End it first."
+            return
+        }
+        #endif
+
         let permitted = await audioCapture.requestPermission()
         guard permitted else {
             errorMessage = "Microphone access is required."
@@ -71,7 +90,7 @@ final class PlaygroundViewModel {
 
         do {
             // AudioCaptureManager dispatches this callback on DispatchQueue.main
-            try audioCapture.startRecording { [weak self] data in
+            try await audioCapture.startRecording { [weak self] data in
                 MainActor.assumeIsolated {
                     guard let self else { return }
                     self.audioBuffer.append(data)
@@ -80,7 +99,10 @@ final class PlaygroundViewModel {
             }
             isRecording = true
             startTimer()
-            logger.info("Recording started")
+            if isAutoStopEnabled {
+                startVADMonitoring()
+            }
+            logger.info("Recording started (autoStop=\(self.isAutoStopEnabled))")
         } catch {
             errorMessage = "Could not start microphone: \(error.localizedDescription)"
             logger.error("Recording start failed: \(error.localizedDescription)")
@@ -91,8 +113,11 @@ final class PlaygroundViewModel {
         audioCapture.stopRecording()
         isRecording = false
         audioLevel = 0
+        speechDetected = false
         timerTask?.cancel()
         timerTask = nil
+        vadMonitorTask?.cancel()
+        vadMonitorTask = nil
 
         guard !audioBuffer.isEmpty else {
             errorMessage = "No audio was captured."
@@ -133,5 +158,63 @@ final class PlaygroundViewModel {
         audioBuffer = Foundation.Data()
         errorMessage = nil
         elapsedSeconds = 0
+        speechDetected = false
+    }
+
+    // MARK: - VAD Monitoring
+
+    private func startVADMonitoring() {
+        vadProcessedBytes = 0
+        hasSpeechBeenDetected = false
+        silenceStartTime = nil
+        speechDetected = false
+
+        vadMonitorTask = Task { [weak self] in
+            while !Task.isCancelled {
+                try? await Task.sleep(nanoseconds: 100_000_000) // 100ms
+                guard let self, !Task.isCancelled, self.isRecording else { break }
+                await self.processVADChunk()
+            }
+        }
+    }
+
+    private func processVADChunk() async {
+        let currentSize = audioBuffer.count
+        guard currentSize > vadProcessedBytes else { return }
+
+        let newData = audioBuffer.subdata(in: vadProcessedBytes..<currentSize)
+        vadProcessedBytes = currentSize
+
+        let samples = convertInt16ToFloat(newData)
+        guard !samples.isEmpty else { return }
+
+        do {
+            let isSpeech = try await RunAnywhere.detectSpeech(in: samples)
+            speechDetected = isSpeech
+
+            if isSpeech {
+                hasSpeechBeenDetected = true
+                silenceStartTime = nil
+            } else if hasSpeechBeenDetected {
+                if silenceStartTime == nil {
+                    silenceStartTime = Date()
+                } else if let start = silenceStartTime,
+                          Date().timeIntervalSince(start) >= autoStopSilenceDuration
+                {
+                    logger.info("Auto-stop: \(self.autoStopSilenceDuration)s silence after speech")
+                    await stopAndTranscribe()
+                }
+            }
+        } catch {
+            logger.error("VAD error: \(error.localizedDescription)")
+        }
+    }
+
+    private func convertInt16ToFloat(_ data: Foundation.Data) -> [Float] {
+        let sampleCount = data.count / MemoryLayout<Int16>.size
+        return data.withUnsafeBytes { rawBuffer in
+            let int16Buffer = rawBuffer.bindMemory(to: Int16.self)
+            return (0..<sampleCount).map { Float(int16Buffer[$0]) / 32768.0 }
+        }
     }
 }