Merge PR #471: VAD model loading infrastructure + iOS VAD playground

Introduces first-class VAD model-loading path in C commons (new
rac_vad_service_t vtable and rac_vad_component_load_model/is_loaded/unload
API), registers ONNX as a VAD provider (Silero), exposes it through the
Swift SDK, ships an iOS demo playground, and bundles YapRun keyboard
refinements.

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

Author: sanchitmonga22

Playground/YapRun/YapRun/Features/Playground/PlaygroundView.swift

@@ -41,6 +41,13 @@ struct PlaygroundView: View {
             }
         }
         .task { await viewModel.checkModelStatus() }
+        .onDisappear {
+            // Release audio resources when leaving the tab to prevent conflicts
+            // with FlowSessionManager's AudioCaptureManager.
+            if viewModel.isRecording {
+                Task { await viewModel.toggleRecording() }
+            }
+        }
         .onReceive(NotificationCenter.default.publisher(for: UIApplication.willEnterForegroundNotification)) { _ in
             Task { await viewModel.checkModelStatus() }
         }
@@ -126,18 +133,38 @@ struct PlaygroundView: View {
                 if viewModel.isRecording {
                     // Recording indicator
                     VStack(spacing: 12) {
-                        // Elapsed time
-                        Text(formatTime(viewModel.elapsedSeconds))
-                            .font(.system(size: 20, weight: .semibold, design: .monospaced))
-                            .foregroundStyle(Color.red)
+                        // Elapsed time + speech indicator
+                        HStack(spacing: 8) {
+                            Text(formatTime(viewModel.elapsedSeconds))
+                                .font(.system(size: 20, weight: .semibold, design: .monospaced))
+                                .foregroundStyle(Color.red)
+
+                            if viewModel.isAutoStopEnabled {
+                                Circle()
+                                    .fill(viewModel.speechDetected ? Color.green : Color.gray.opacity(0.4))
+                                    .frame(width: 8, height: 8)
+                            }
+                        }
 
                         // Waveform bars
                         WaveformBars(level: viewModel.audioLevel)
                     }
                 } else {
-                    Text(viewModel.transcription.isEmpty ? "Tap to record" : "Tap to record again")
-                        .font(.subheadline)
-                        .foregroundStyle(AppColors.textTertiary)
+                    VStack(spacing: 12) {
+                        Text(viewModel.transcription.isEmpty ? "Tap to record" : "Tap to record again")
+                            .font(.subheadline)
+                            .foregroundStyle(AppColors.textTertiary)
+
+                        // Auto-stop toggle
+                        Toggle(isOn: $viewModel.isAutoStopEnabled) {
+                            Label("Auto-stop on silence", systemImage: "waveform.badge.minus")
+                                .font(.caption)
+                                .foregroundStyle(AppColors.textSecondary)
+                        }
+                        .toggleStyle(.switch)
+                        .tint(AppColors.primaryGreen)
+                        .frame(width: 240)
+                    }
                 }
             }
         }

Playground/YapRun/YapRun/Features/Playground/PlaygroundViewModel.swift

@@ -25,11 +25,21 @@ final class PlaygroundViewModel {
     var errorMessage: String?
     var modelName: String?
 
+    // MARK: - Auto-Stop (VAD)
+
+    var isAutoStopEnabled = false
+    var speechDetected = false
+
     // MARK: - Private
 
     private let audioCapture = AudioCaptureManager()
     private var audioBuffer = Foundation.Data()
     private var timerTask: Task<Void, Never>?
+    private var vadMonitorTask: Task<Void, Never>?
+    private var vadProcessedBytes = 0
+    private var silenceStartTime: Date?
+    private var hasSpeechBeenDetected = false
+    private let autoStopSilenceDuration: TimeInterval = 2.0
     private let logger = Logger(subsystem: "com.runanywhere.yaprun", category: "Playground")
 
     // MARK: - Model Check
@@ -58,6 +68,15 @@ final class PlaygroundViewModel {
             return
         }
 
+        // Prevent conflict with active voice keyboard session (iOS-only).
+        // FlowSessionManager is compiled `#if os(iOS)` so this check is skipped on macOS.
+        #if os(iOS)
+        guard !FlowSessionManager.shared.isActive else {
+            errorMessage = "Voice keyboard session is active. End it first."
+            return
+        }
+        #endif
+
         let permitted = await audioCapture.requestPermission()
         guard permitted else {
             errorMessage = "Microphone access is required."
@@ -71,7 +90,7 @@ final class PlaygroundViewModel {
 
         do {
             // AudioCaptureManager dispatches this callback on DispatchQueue.main
-            try audioCapture.startRecording { [weak self] data in
+            try await audioCapture.startRecording { [weak self] data in
                 MainActor.assumeIsolated {
                     guard let self else { return }
                     self.audioBuffer.append(data)
@@ -80,7 +99,10 @@ final class PlaygroundViewModel {
             }
             isRecording = true
             startTimer()
-            logger.info("Recording started")
+            if isAutoStopEnabled {
+                startVADMonitoring()
+            }
+            logger.info("Recording started (autoStop=\(self.isAutoStopEnabled))")
         } catch {
             errorMessage = "Could not start microphone: \(error.localizedDescription)"
             logger.error("Recording start failed: \(error.localizedDescription)")
@@ -91,8 +113,11 @@ final class PlaygroundViewModel {
         audioCapture.stopRecording()
         isRecording = false
         audioLevel = 0
+        speechDetected = false
         timerTask?.cancel()
         timerTask = nil
+        vadMonitorTask?.cancel()
+        vadMonitorTask = nil
 
         guard !audioBuffer.isEmpty else {
             errorMessage = "No audio was captured."
@@ -133,5 +158,63 @@ final class PlaygroundViewModel {
         audioBuffer = Foundation.Data()
         errorMessage = nil
         elapsedSeconds = 0
+        speechDetected = false
+    }
+
+    // MARK: - VAD Monitoring
+
+    private func startVADMonitoring() {
+        vadProcessedBytes = 0
+        hasSpeechBeenDetected = false
+        silenceStartTime = nil
+        speechDetected = false
+
+        vadMonitorTask = Task { [weak self] in
+            while !Task.isCancelled {
+                try? await Task.sleep(nanoseconds: 100_000_000) // 100ms
+                guard let self, !Task.isCancelled, self.isRecording else { break }
+                await self.processVADChunk()
+            }
+        }
+    }
+
+    private func processVADChunk() async {
+        let currentSize = audioBuffer.count
+        guard currentSize > vadProcessedBytes else { return }
+
+        let newData = audioBuffer.subdata(in: vadProcessedBytes..<currentSize)
+        vadProcessedBytes = currentSize
+
+        let samples = convertInt16ToFloat(newData)
+        guard !samples.isEmpty else { return }
+
+        do {
+            let isSpeech = try await RunAnywhere.detectSpeech(in: samples)
+            speechDetected = isSpeech
+
+            if isSpeech {
+                hasSpeechBeenDetected = true
+                silenceStartTime = nil
+            } else if hasSpeechBeenDetected {
+                if silenceStartTime == nil {
+                    silenceStartTime = Date()
+                } else if let start = silenceStartTime,
+                          Date().timeIntervalSince(start) >= autoStopSilenceDuration
+                {
+                    logger.info("Auto-stop: \(self.autoStopSilenceDuration)s silence after speech")
+                    await stopAndTranscribe()
+                }
+            }
+        } catch {
+            logger.error("VAD error: \(error.localizedDescription)")
+        }
+    }
+
+    private func convertInt16ToFloat(_ data: Foundation.Data) -> [Float] {
+        let sampleCount = data.count / MemoryLayout<Int16>.size
+        return data.withUnsafeBytes { rawBuffer in
+            let int16Buffer = rawBuffer.bindMemory(to: Int16.self)
+            return (0..<sampleCount).map { Float(int16Buffer[$0]) / 32768.0 }
+        }
     }
 }

Playground/YapRun/YapRun/Features/VoiceKeyboard/FlowSessionManager.swift

@@ -159,7 +159,7 @@ final class FlowSessionManager: ObservableObject {
         // Start AVAudioEngine while foregrounded
         do {
             // AudioCaptureManager dispatches this callback on DispatchQueue.main
-            try audioCapture.startRecording { [weak self] data in
+            try await audioCapture.startRecording { [weak self] data in
                 MainActor.assumeIsolated {
                     guard let self else { return }
                     SharedDataBridge.shared.audioLevel = self.audioCapture.audioLevel
@@ -191,6 +191,7 @@ final class FlowSessionManager: ObservableObject {
             logger.warning("startListening received in unexpected phase: \(self.sessionPhase.description)")
             return
         }
+        logger.info("startListening received; transitioning to listening")
         audioBuffer = Data()
         transition(to: .listening)
         SharedDataBridge.shared.sessionState = "listening"
@@ -206,6 +207,7 @@ final class FlowSessionManager: ObservableObject {
             return
         }
 
+        logger.info("stopListening received; transitioning to transcribing")
         transition(to: .transcribing)
         SharedDataBridge.shared.sessionState = "transcribing"
         if #available(iOS 16.1, *) {
@@ -251,8 +253,20 @@ final class FlowSessionManager: ObservableObject {
         logger.info("Transcribing \(audio.count) bytes")
 
         do {
-            let text = try await RunAnywhere.transcribe(audio)
+            let text = try await Task.detached(priority: .userInitiated) {
+                try await RunAnywhere.transcribe(audio)
+            }.value
             logger.info("Transcription complete: \"\(text)\"")
+
+            // Task.detached drops structured cancellation, so endSession()/killSession()
+            // cannot cancel the transcription in flight. If the session was torn down
+            // while transcription was running, discard the result rather than writing
+            // it to SharedDataBridge after the session is idle.
+            guard case .transcribing = sessionPhase else {
+                logger.info("Session no longer transcribing — discarding result")
+                return
+            }
+
             wordCount += text.split(separator: " ").count
 
             if #available(iOS 16.1, *) {
@@ -277,9 +291,15 @@ final class FlowSessionManager: ObservableObject {
         SharedDataBridge.shared.lastInsertedText = text
         SharedDataBridge.shared.sessionState = "done"
 
-        DarwinNotificationCenter.shared.post(
-            name: SharedConstants.DarwinNotifications.transcriptionReady
-        )
+        // Small delay to allow cross-process UserDefaults propagation before posting
+        // the Darwin notification. Modern iOS syncs UserDefaults automatically within
+        // ~100ms — the 50ms delay + notification timing accounts for this.
+        Task {
+            try? await Task.sleep(nanoseconds: 50_000_000) // 50ms
+            DarwinNotificationCenter.shared.post(
+                name: SharedConstants.DarwinNotifications.transcriptionReady
+            )
+        }
 
         transition(to: .done(text))
         appendHistory(text: text)

Playground/YapRun/YapRun/Shared/SharedConstants.swift

@@ -39,6 +39,8 @@ enum SharedConstants {
         // app → keyboard
         static let transcriptionReady = "com.runanywhere.yaprun.keyboard.transcriptionReady"
         static let sessionReady       = "com.runanywhere.yaprun.session.ready"
+        static let stateChanged       = "com.runanywhere.yaprun.session.stateChanged"
+        static let audioLevelChanged  = "com.runanywhere.yaprun.audioLevelChanged"
         // keyboard → app
         static let startListening     = "com.runanywhere.yaprun.keyboard.startListening"
         static let stopListening      = "com.runanywhere.yaprun.keyboard.stopListening"

Playground/YapRun/YapRun/Shared/SharedDataBridge.swift

@@ -74,20 +74,22 @@ final class SharedDataBridge {
 
     var sessionState: String {
         get {
-            defaults?.synchronize()
             return defaults?.string(forKey: SharedConstants.Keys.sessionState) ?? "idle"
         }
         set {
             defaults?.set(newValue, forKey: SharedConstants.Keys.sessionState)
-            defaults?.synchronize()
+            // Push-notify the keyboard extension (or main app) immediately so it
+            // doesn't have to wait for the next poll timer tick to see the change.
+            DarwinNotificationCenter.shared.post(
+                name: SharedConstants.DarwinNotifications.stateChanged
+            )
         }
     }
 
     // MARK: - Transcription Result
 
     var transcribedText: String? {
         get {
-            defaults?.synchronize()
             return defaults?.string(forKey: SharedConstants.Keys.transcribedText)
         }
         set {
@@ -96,7 +98,6 @@ final class SharedDataBridge {
             } else {
                 defaults?.removeObject(forKey: SharedConstants.Keys.transcribedText)
             }
-            defaults?.synchronize()
         }
     }
 
@@ -118,7 +119,14 @@ final class SharedDataBridge {
 
     var audioLevel: Float {
         get { defaults?.float(forKey: SharedConstants.Keys.audioLevel) ?? 0 }
-        set { defaults?.set(newValue, forKey: SharedConstants.Keys.audioLevel) }
+        set {
+            defaults?.set(newValue, forKey: SharedConstants.Keys.audioLevel)
+            // Push-notify the keyboard extension so it can read the cached value
+            // instead of polling UserDefaults on every waveform tick.
+            DarwinNotificationCenter.shared.post(
+                name: SharedConstants.DarwinNotifications.audioLevelChanged
+            )
+        }
     }
 
     // MARK: - Heartbeat

Playground/YapRun/YapRun/macOS/Services/MacDictationService.swift

@@ -166,7 +166,7 @@ final class MacDictationService {
 
         do {
             // AudioCaptureManager dispatches this callback on DispatchQueue.main
-            try audioCapture.startRecording { [weak self] data in
+            try await audioCapture.startRecording { [weak self] data in
                 MainActor.assumeIsolated {
                     guard let self else { return }
                     self.audioBuffer.append(data)