Thinking mode in rag plus audio, also fixed camera

VyasGuru · sanchitmonga22 · commit b2e94a34c454 · 2026-04-14T13:07:01.000-07:00
diff --git a/examples/ios/RunAnywhereAI/RunAnywhereAI/App/RunAnywhereAIApp.swift b/examples/ios/RunAnywhereAI/RunAnywhereAI/App/RunAnywhereAIApp.swift
@@ -64,6 +64,7 @@ struct RunAnywhereAIApp: App {
                 }
             }
             .task {
+                _ = SettingsViewModel.shared
                 logger.info("🏁 App launched, initializing SDK...")
                 await initializeSDK()
             }
diff --git a/examples/ios/RunAnywhereAI/RunAnywhereAI/Features/Chat/ViewModels/LLMViewModel+Generation.swift b/examples/ios/RunAnywhereAI/RunAnywhereAI/Features/Chat/ViewModels/LLMViewModel+Generation.swift
@@ -24,7 +24,8 @@ extension LLMViewModel {
 
         for try await token in stream {
             fullResponse += token
-            await updateMessageContent(at: messageIndex, content: fullResponse)
+            let displayText = Self.stripThinkTags(from: fullResponse)
+            await updateMessageContent(at: messageIndex, content: displayText)
             NotificationCenter.default.post(
                 name: Notification.Name("MessageContentUpdated"),
                 object: nil
diff --git a/examples/ios/RunAnywhereAI/RunAnywhereAI/Features/Chat/ViewModels/LLMViewModel.swift b/examples/ios/RunAnywhereAI/RunAnywhereAI/Features/Chat/ViewModels/LLMViewModel.swift
@@ -574,4 +574,19 @@ final class LLMViewModel {
             loadConversation(conversation)
         }
     }
+
+    static func stripThinkTags(from text: String) -> String {
+        var result = text
+        // Remove complete <think>...</think> blocks 
+        while let startRange = result.range(of: "<think>"),
+              let endRange = result.range(of: "</think>"),
+              startRange.upperBound <= endRange.lowerBound {
+            result.removeSubrange(startRange.lowerBound..<endRange.upperBound)
+        }
+        if let trailingStart = result.range(of: "<think>", options: .backwards),
+           result.range(of: "</think>", range: trailingStart.upperBound..<result.endIndex) == nil {
+            result = String(result[result.startIndex..<trailingStart.lowerBound])
+        }
+        return result.trimmingCharacters(in: .whitespacesAndNewlines)
+    }
 }
diff --git a/examples/ios/RunAnywhereAI/RunAnywhereAI/Features/Chat/Views/ChatInterfaceView.swift b/examples/ios/RunAnywhereAI/RunAnywhereAI/Features/Chat/Views/ChatInterfaceView.swift
@@ -370,8 +370,8 @@ extension ChatInterfaceView {
             .onReceive(
                 NotificationCenter.default.publisher(for: Notification.Name("MessageContentUpdated"))
             ) { _ in
-                if viewModel.isGenerating {
-                    proxy.scrollTo("typing", anchor: .bottom)
+                if viewModel.isGenerating, let lastMessage = viewModel.messages.last {
+                    proxy.scrollTo(lastMessage.id, anchor: .bottom)
                 }
             }
         }
@@ -413,7 +413,7 @@ extension ChatInterfaceView {
                     .animation(nil, value: message.content)
             }
 
-            if viewModel.isGenerating {
+            if viewModel.isGenerating, viewModel.messages.last?.content.isEmpty == true {
                 TypingIndicatorView()
                     .id("typing")
                     .transition(typingTransition)
diff --git a/examples/ios/RunAnywhereAI/RunAnywhereAI/Features/RAG/ViewModels/RAGViewModel.swift b/examples/ios/RunAnywhereAI/RunAnywhereAI/Features/RAG/ViewModels/RAGViewModel.swift
@@ -107,8 +107,16 @@ final class RAGViewModel {
         }
 
         do {
+            let settings = SettingsViewModel.shared
+            let effectiveQuestion: String
+            if settings.loadedModelSupportsThinking && !settings.thinkingModeEnabled {
+                effectiveQuestion = "/no_think\n\(question)"
+            } else {
+                effectiveQuestion = question
+            }
+
             logger.info("Querying RAG pipeline: \(question)")
-            let result = try await RunAnywhere.ragQuery(question: question)
+            let result = try await RunAnywhere.ragQuery(question: effectiveQuestion)
             messages.append((role: .assistant, text: result.answer))
             logger.info("Query complete (\(result.totalTimeMs, format: .fixed(precision: 0))ms)")
         } catch {
diff --git a/examples/ios/RunAnywhereAI/RunAnywhereAI/Features/Settings/SettingsViewModel.swift b/examples/ios/RunAnywhereAI/RunAnywhereAI/Features/Settings/SettingsViewModel.swift
@@ -194,6 +194,10 @@ class SettingsViewModel: ObservableObject {
 
         // Load system prompt — fall back to the default when the key has never been set
         systemPrompt = UserDefaults.standard.string(forKey: systemPromptDefaultsKey) ?? "You are a helpful, concise AI assistant."
+        // Persist the default so that other ViewModels reading UserDefaults directly always find a value
+        if UserDefaults.standard.string(forKey: systemPromptDefaultsKey) == nil {
+            UserDefaults.standard.set(systemPrompt, forKey: systemPromptDefaultsKey)
+        }
 
         // Load thinking mode
         thinkingModeEnabled = UserDefaults.standard.bool(forKey: thinkingModeKey)
diff --git a/examples/ios/RunAnywhereAI/RunAnywhereAI/Features/Settings/ToolSettingsView.swift b/examples/ios/RunAnywhereAI/RunAnywhereAI/Features/Settings/ToolSettingsView.swift
@@ -84,7 +84,12 @@ class ToolSettingsViewModel: ObservableObject {
                     category: "Utility"
                 ),
                 executor: { args in
-                    let expression = args["expression"]?.stringValue ?? args["input"]?.stringValue ?? "0"
+                    let expression = args["expression"]?.stringValue
+                        ?? args["input"]?.stringValue
+                        ?? args["expr"]?.stringValue
+                        ?? args.values.compactMap(\.stringValue).first
+                        ?? "0"
+                    print("Calculator received args: \(args), using expression: '\(expression)'")
                     // Clean the expression - remove any non-math characters
                     let cleanedExpression = expression
                         .replacingOccurrences(of: "=", with: "")
diff --git a/examples/ios/RunAnywhereAI/RunAnywhereAI/Features/Vision/VLMCameraView.swift b/examples/ios/RunAnywhereAI/RunAnywhereAI/Features/Vision/VLMCameraView.swift
@@ -296,8 +296,10 @@ struct VLMCameraView: View {
     private func setupCameraIfNeeded() {
         Task {
             await viewModel.checkCameraAuthorization()
-            if viewModel.isCameraAuthorized && viewModel.captureSession == nil {
-                viewModel.setupCamera()
+            if viewModel.isCameraAuthorized {
+                if viewModel.captureSession == nil {
+                    viewModel.setupCamera()
+                }
                 viewModel.startCamera()
             }
         }
diff --git a/examples/ios/RunAnywhereAI/RunAnywhereAI/Features/Voice/VoiceAgentViewModel.swift b/examples/ios/RunAnywhereAI/RunAnywhereAI/Features/Voice/VoiceAgentViewModel.swift
@@ -387,7 +387,11 @@ final class VoiceAgentViewModel: ObservableObject {
         assistantResponse = ""
 
         do {
-            session = try await RunAnywhere.startVoiceSession()
+            let settings = SettingsViewModel.shared
+            let voiceConfig = VoiceSessionConfig(
+                thinkingModeEnabled: settings.loadedModelSupportsThinking && settings.thinkingModeEnabled
+            )
+            session = try await RunAnywhere.startVoiceSession(config: voiceConfig)
             sessionState = .listening
             currentStatus = "Listening..."
             eventTask = Task { [weak self] in
@@ -419,6 +423,10 @@ final class VoiceAgentViewModel: ObservableObject {
         logger.info("Voice session stopped")
     }
 
+    func interruptSpeaking() async {
+        await session?.interruptPlayback()
+    }
+
     /// Force send current audio buffer (for push-to-talk mode)
     func sendAudioNow() async {
         await session?.sendNow()
@@ -434,9 +442,9 @@ final class VoiceAgentViewModel: ObservableObject {
         case .speechStarted: isSpeechDetected = true; currentStatus = "Listening..."
         case .processing: sessionState = .processing; currentStatus = "Processing..."; isSpeechDetected = false
         case .transcribed(let text): currentTranscript = text
-        case .responded(let text): assistantResponse = text
+        case .responded(let text, _): assistantResponse = text
         case .speaking: sessionState = .speaking; currentStatus = "Speaking..."
-        case let .turnCompleted(transcript, response, _):
+        case let .turnCompleted(transcript, response, _, _):
             currentTranscript = transcript; assistantResponse = response
             sessionState = .listening; currentStatus = "Listening..."
         case .stopped: sessionState = .disconnected; currentStatus = "Ready"
diff --git a/examples/ios/RunAnywhereAI/RunAnywhereAI/Features/Voice/VoiceAssistantView.swift b/examples/ios/RunAnywhereAI/RunAnywhereAI/Features/Voice/VoiceAssistantView.swift
@@ -421,7 +421,9 @@ extension VoiceAssistantView {
                 icon: viewModel.micButtonIcon
             ) {
                 Task {
-                    if viewModel.isActive {
+                    if viewModel.isSpeaking {
+                        await viewModel.interruptSpeaking()
+                    } else if viewModel.isActive {
                         await viewModel.stopConversation()
                     } else {
                         await viewModel.startConversation()
diff --git a/sdk/runanywhere-swift/Sources/RunAnywhere/Features/TTS/Services/AudioPlaybackManager.swift b/sdk/runanywhere-swift/Sources/RunAnywhere/Features/TTS/Services/AudioPlaybackManager.swift
@@ -87,7 +87,7 @@ public class AudioPlaybackManager: NSObject, ObservableObject, AVAudioPlayerDele
 
     /// Stop current playback
     public func stop() {
-        guard isPlaying else { return }
+        guard audioPlayer != nil else { return }
 
         audioPlayer?.stop()
         cleanupPlayback(success: false)
diff --git a/sdk/runanywhere-swift/Sources/RunAnywhere/Public/Extensions/LLM/RunAnywhere+TextGeneration.swift b/sdk/runanywhere-swift/Sources/RunAnywhere/Public/Extensions/LLM/RunAnywhere+TextGeneration.swift
@@ -309,7 +309,7 @@ private final class LLMStreamCallbackContext: @unchecked Sendable {
 
 // MARK: - Thinking Content Parser
 
-private enum ThinkingContentParser {
+enum ThinkingContentParser {
     /// Extracts `<think>...</think>` content from generated text.
     /// - Returns: Tuple of (responseText, thinkingContent). If no tags found, responseText = original text, thinkingContent = nil.
     static func extract(from text: String) -> (text: String, thinking: String?) {
@@ -320,10 +320,16 @@ private enum ThinkingContentParser {
         }
         let thinkingContent = String(text[startRange.upperBound..<endRange.lowerBound])
             .trimmingCharacters(in: .whitespacesAndNewlines)
-        let responseText = String(text[endRange.upperBound...])
+        // Include any text before <think> and after </think>
+        let textBefore = String(text[..<startRange.lowerBound])
             .trimmingCharacters(in: .whitespacesAndNewlines)
+        let textAfter = String(text[endRange.upperBound...])
+            .trimmingCharacters(in: .whitespacesAndNewlines)
+        let responseText = [textBefore, textAfter]
+            .filter { !$0.isEmpty }
+            .joined(separator: "\n")
         return (
-            text: responseText.isEmpty ? text : responseText,
+            text: responseText,
             thinking: thinkingContent.isEmpty ? nil : thinkingContent
         )
     }
diff --git a/sdk/runanywhere-swift/Sources/RunAnywhere/Public/Extensions/VoiceAgent/RunAnywhere+VoiceAgent.swift b/sdk/runanywhere-swift/Sources/RunAnywhere/Public/Extensions/VoiceAgent/RunAnywhere+VoiceAgent.swift
@@ -257,10 +257,14 @@ public extension RunAnywhere {
             rac_voice_agent_synthesize_speech(handle, textPtr, &audioPtr, &audioSize)
         }
 
-        guard result == RAC_SUCCESS, let ptr = audioPtr, audioSize > 0 else {
+        guard result == RAC_SUCCESS else {
             throw SDKError.voiceAgent(.processingFailed, "Speech synthesis failed: \(result)")
         }
 
+        guard let ptr = audioPtr, audioSize > 0 else {
+            return Data()
+        }
+
         let audioData = Data(bytes: ptr, count: audioSize)
         free(ptr)
 
diff --git a/sdk/runanywhere-swift/Sources/RunAnywhere/Public/Extensions/VoiceAgent/RunAnywhere+VoiceSession.swift b/sdk/runanywhere-swift/Sources/RunAnywhere/Public/Extensions/VoiceAgent/RunAnywhere+VoiceSession.swift
@@ -108,6 +108,10 @@ public actor VoiceSessionHandle {
         eventContinuation?.finish()
     }
 
+    public func interruptPlayback() {
+        audioPlayback.stop()
+    }
+
     /// Force process current audio (push-to-talk)
     public func sendNow() async {
         guard isRunning else { return }
@@ -196,44 +200,69 @@ public actor VoiceSessionHandle {
 
         emit(.processing)
 
+        var transcription = ""
+        var cleanedResponse = ""
+        var thinkingContent: String?
+        var synthesizedAudio: Data?
+
         do {
-            let result = try await RunAnywhere.processVoiceTurn(audio)
+            // Step 1: Transcribe audio
+            transcription = try await RunAnywhere.voiceAgentTranscribe(audio)
 
-            guard result.speechDetected else {
-                logger.info("No speech detected")
+            guard !transcription.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty else {
+                logger.info("No speech detected (empty transcription)")
+                emit(.turnCompleted(transcript: "", response: "", thinkingContent: nil, audio: nil))
                 if config.continuousMode && isRunning {
                     try? await startListening()
                 }
                 return
             }
 
-            // Emit intermediate results
-            if let transcript = result.transcription {
-                emit(.transcribed(text: transcript))
-            }
+            emit(.transcribed(text: transcription))
 
-            if let response = result.response {
-                emit(.responded(text: response))
+            // Step 2: Generate LLM response (apply /no_think prefix if needed)
+            let effectivePrompt: String
+            if !config.thinkingModeEnabled {
+                effectivePrompt = "/no_think\n\(transcription)"
+            } else {
+                effectivePrompt = transcription
             }
 
-            // Play TTS if enabled
-            if config.autoPlayTTS, let ttsAudio = result.synthesizedAudio, !ttsAudio.isEmpty {
-                emit(.speaking)
-                try await audioPlayback.play(ttsAudio)
-            }
+            let rawResponse = try await RunAnywhere.voiceAgentGenerateResponse(effectivePrompt)
 
-            // Emit complete result
-            emit(.turnCompleted(
-                transcript: result.transcription ?? "",
-                response: result.response ?? "",
-                audio: result.synthesizedAudio
-            ))
+            // Step 3: Parse out <think> tags from response before TTS
+            let parsed = ThinkingContentParser.extract(from: rawResponse)
+            cleanedResponse = parsed.text
+            thinkingContent = parsed.thinking
 
+            emit(.responded(text: cleanedResponse, thinkingContent: thinkingContent))
+
+            // Step 4: Synthesize speech from cleaned response (no think tags spoken)
+            if config.autoPlayTTS, !cleanedResponse.isEmpty {
+                let ttsAudio = try await RunAnywhere.voiceAgentSynthesizeSpeech(cleanedResponse)
+                synthesizedAudio = ttsAudio
+
+                if !ttsAudio.isEmpty {
+                    emit(.speaking)
+                    do {
+                        try await audioPlayback.play(ttsAudio)
+                    } catch is AudioPlaybackError {
+                        logger.info("TTS playback interrupted by user")
+                    }
+                }
+            }
         } catch {
             logger.error("Processing failed: \(error)")
             emit(.error(error.localizedDescription))
         }
 
+        emit(.turnCompleted(
+            transcript: transcription,
+            response: cleanedResponse,
+            thinkingContent: thinkingContent,
+            audio: synthesizedAudio
+        ))
+
         // Resume listening if continuous mode
         if config.continuousMode && isRunning {
             try? await startListening()
diff --git a/sdk/runanywhere-swift/Sources/RunAnywhere/Public/Extensions/VoiceAgent/VoiceAgentTypes.swift b/sdk/runanywhere-swift/Sources/RunAnywhere/Public/Extensions/VoiceAgent/VoiceAgentTypes.swift
@@ -23,6 +23,9 @@ public struct VoiceAgentResult: Sendable {
     /// Generated response text from LLM
     public var response: String?
 
+    /// Thinking content extracted from `<think>...</think>` tags (nil if none)
+    public var thinkingContent: String?
+
     /// Synthesized audio data from TTS
     public var synthesizedAudio: Data?
 
@@ -31,11 +34,13 @@ public struct VoiceAgentResult: Sendable {
         speechDetected: Bool = false,
         transcription: String? = nil,
         response: String? = nil,
+        thinkingContent: String? = nil,
         synthesizedAudio: Data? = nil
     ) {
         self.speechDetected = speechDetected
         self.transcription = transcription
         self.response = response
+        self.thinkingContent = thinkingContent
         self.synthesizedAudio = synthesizedAudio
     }
 
@@ -185,14 +190,14 @@ public enum VoiceSessionEvent: Sendable {
     /// Got transcription from STT
     case transcribed(text: String)
 
-    /// Got response from LLM
-    case responded(text: String)
+    /// Got response from LLM (with optional thinking content)
+    case responded(text: String, thinkingContent: String? = nil)
 
     /// Playing TTS audio
     case speaking
 
-    /// Complete turn result
-    case turnCompleted(transcript: String, response: String, audio: Data?)
+    /// Complete turn result (with optional thinking content)
+    case turnCompleted(transcript: String, response: String, thinkingContent: String? = nil, audio: Data?)
 
     /// Session stopped
     case stopped
@@ -217,16 +222,21 @@ public struct VoiceSessionConfig: Sendable {
     /// Whether to auto-resume listening after TTS playback
     public var continuousMode: Bool
 
+    /// Whether thinking mode is enabled for the LLM.
+    public var thinkingModeEnabled: Bool
+
     public init(
         silenceDuration: TimeInterval = 1.5,
         speechThreshold: Float = 0.1,
         autoPlayTTS: Bool = true,
-        continuousMode: Bool = true
+        continuousMode: Bool = true,
+        thinkingModeEnabled: Bool = false
     ) {
         self.silenceDuration = silenceDuration
         self.speechThreshold = speechThreshold
         self.autoPlayTTS = autoPlayTTS
         self.continuousMode = continuousMode
+        self.thinkingModeEnabled = thinkingModeEnabled
     }
 
     /// Default configuration

Original file line number	Diff line number	Diff line change
`@@ -64,6 +64,7 @@ struct RunAnywhereAIApp: App {`
`64`	`64`	`}`
`65`	`65`	`}`
`66`	`66`	`.task {`
	`67`	`+ _ = SettingsViewModel.shared`
`67`	`68`	`logger.info("🏁 App launched, initializing SDK...")`
`68`	`69`	`await initializeSDK()`
`69`	`70`	`}`
Original file line number	Diff line number	Diff line change
`@@ -370,8 +370,8 @@ extension ChatInterfaceView {`
`370`	`370`	`.onReceive(`
`371`	`371`	`NotificationCenter.default.publisher(for: Notification.Name("MessageContentUpdated"))`
`372`	`372`	`) { _ in`
`373`		`- if viewModel.isGenerating {`
`374`		`- proxy.scrollTo("typing", anchor: .bottom)`
	`373`	`+ if viewModel.isGenerating, let lastMessage = viewModel.messages.last {`
	`374`	`+ proxy.scrollTo(lastMessage.id, anchor: .bottom)`
`375`	`375`	`}`
`376`	`376`	`}`
`377`	`377`	`}`
`@@ -413,7 +413,7 @@ extension ChatInterfaceView {`
`413`	`413`	`.animation(nil, value: message.content)`
`414`	`414`	`}`
`415`	`415`
`416`		`- if viewModel.isGenerating {`
	`416`	`+ if viewModel.isGenerating, viewModel.messages.last?.content.isEmpty == true {`
`417`	`417`	`TypingIndicatorView()`
`418`	`418`	`.id("typing")`
`419`	`419`	`.transition(typingTransition)`
Original file line number	Diff line number	Diff line change
`@@ -296,8 +296,10 @@ struct VLMCameraView: View {`
`296`	`296`	`private func setupCameraIfNeeded() {`
`297`	`297`	`Task {`
`298`	`298`	`await viewModel.checkCameraAuthorization()`
`299`		`- if viewModel.isCameraAuthorized && viewModel.captureSession == nil {`
`300`		`- viewModel.setupCamera()`
	`299`	`+ if viewModel.isCameraAuthorized {`
	`300`	`+ if viewModel.captureSession == nil {`
	`301`	`+ viewModel.setupCamera()`
	`302`	`+ }`
`301`	`303`	`viewModel.startCamera()`
`302`	`304`	`}`
`303`	`305`	`}`