Skip to content

Commit 0fae755

Browse files
sanchitmonga22sanchitmonga22
committed
updates to whisper kit
1 parent fc64f8b commit 0fae755

15 files changed

Lines changed: 863 additions & 130 deletions

File tree

‎.gitignore‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -383,3 +383,4 @@ sdk/runanywhere-react-native/packages/*/ios/xcframeworks/
383383
# Node
384384
node_modules/
385385
/tools/
386+
*.trace

‎sdk/runanywhere-commons/CMakeLists.txt‎

Lines changed: 19 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,11 @@ option(RAC_BACKEND_LLAMACPP "Build LlamaCPP backend" ON)
3434
option(RAC_BACKEND_ONNX "Build ONNX backend" ON)
3535
# WhisperCPP OFF by default - Sherpa-ONNX (NeMo CTC / Parakeet) is now the primary STT backend
3636
option(RAC_BACKEND_WHISPERCPP "Build WhisperCPP backend" OFF)
37+
if(APPLE)
38+
option(RAC_BACKEND_WHISPERKIT_COREML "Build WhisperKit CoreML backend (Apple Neural Engine STT)" ON)
39+
else()
40+
set(RAC_BACKEND_WHISPERKIT_COREML OFF CACHE BOOL "" FORCE)
41+
endif()
3742
option(RAC_BUILD_SERVER "Build OpenAI-compatible HTTP server (runanywhere-server)" OFF)
3843

3944
# =============================================================================
@@ -263,12 +268,24 @@ else()
263268
set(RAC_PLATFORM_SOURCES "")
264269
endif()
265270

271+
# WhisperKit CoreML backend (Apple-only, no external deps → compiled into rac_commons)
272+
if(APPLE AND RAC_BACKEND_WHISPERKIT_COREML)
273+
set(RAC_WHISPERKIT_COREML_SOURCES
274+
src/backends/whisperkit_coreml/rac_stt_whisperkit_coreml.cpp
275+
src/backends/whisperkit_coreml/rac_backend_whisperkit_coreml_register.cpp
276+
)
277+
message(STATUS "Building WhisperKit CoreML backend (compiled into rac_commons)")
278+
else()
279+
set(RAC_WHISPERKIT_COREML_SOURCES "")
280+
endif()
281+
266282
# Combine all sources
267283
set(RAC_COMMONS_SOURCES
268284
${RAC_CORE_SOURCES}
269285
${RAC_INFRASTRUCTURE_SOURCES}
270286
${RAC_FEATURES_SOURCES}
271287
${RAC_PLATFORM_SOURCES}
288+
${RAC_WHISPERKIT_COREML_SOURCES}
272289
)
273290

274291
# =============================================================================
@@ -378,6 +395,7 @@ if(RAC_BUILD_BACKENDS)
378395
message(STATUS " - WhisperCPP backend")
379396
add_subdirectory(src/backends/whispercpp)
380397
endif()
398+
381399
endif()
382400

383401
# =============================================================================
@@ -443,7 +461,7 @@ if(APPLE AND RAC_BUILD_PLATFORM)
443461
message(STATUS " Platform: Apple Foundation Models, System TTS")
444462
endif()
445463
if(RAC_BUILD_BACKENDS)
446-
message(STATUS " Backends: LlamaCPP=${RAC_BACKEND_LLAMACPP}, ONNX=${RAC_BACKEND_ONNX}, WhisperCPP=${RAC_BACKEND_WHISPERCPP}")
464+
message(STATUS " Backends: LlamaCPP=${RAC_BACKEND_LLAMACPP}, ONNX=${RAC_BACKEND_ONNX}, WhisperCPP=${RAC_BACKEND_WHISPERCPP}, WhisperKitCoreML=${RAC_BACKEND_WHISPERKIT_COREML}")
447465
endif()
448466
if(RAC_BUILD_SERVER)
449467
message(STATUS " Server: OpenAI-compatible HTTP server (runanywhere-server)")

‎sdk/runanywhere-commons/include/rac/backends/rac_stt_whisperkit_coreml.h‎

Lines changed: 138 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,138 @@
1+
/**
2+
* @file rac_stt_whisperkit_coreml.h
3+
* @brief RunAnywhere Commons - WhisperKit CoreML STT Backend (Apple Neural Engine)
4+
*
5+
* C API for the WhisperKit CoreML STT backend. The actual inference runs in
6+
* Swift via WhisperKit + CoreML; C++ provides the callback infrastructure,
7+
* vtable dispatch, and automatic telemetry through the standard stt_component
8+
* pipeline.
9+
*
10+
* This backend is Apple-only. On non-Apple platforms it is never registered.
11+
*/
12+
13+
#ifndef RAC_STT_WHISPERKIT_COREML_H
14+
#define RAC_STT_WHISPERKIT_COREML_H
15+
16+
#include "rac/core/rac_types.h"
17+
#include "rac/features/stt/rac_stt_types.h"
18+
19+
#ifdef __cplusplus
20+
extern "C" {
21+
#endif
22+
23+
// =============================================================================
24+
// SWIFT CALLBACK TYPES
25+
// =============================================================================
26+
27+
/**
28+
* Callback to check if WhisperKit CoreML can handle a model ID.
29+
*
30+
* @param model_id Model identifier to check (can be NULL)
31+
* @param user_data User-provided context
32+
* @return RAC_TRUE if WhisperKit CoreML can handle this model
33+
*/
34+
typedef rac_bool_t (*rac_whisperkit_coreml_stt_can_handle_fn)(const char* model_id,
35+
void* user_data);
36+
37+
/**
38+
* Callback to load a WhisperKit CoreML model.
39+
*
40+
* @param model_path Path to model directory containing .mlmodelc files
41+
* @param model_id Model identifier
42+
* @param user_data User-provided context
43+
* @return Opaque handle to loaded service, or NULL on failure
44+
*/
45+
typedef rac_handle_t (*rac_whisperkit_coreml_stt_create_fn)(const char* model_path,
46+
const char* model_id,
47+
void* user_data);
48+
49+
/**
50+
* Callback to transcribe audio via WhisperKit CoreML.
51+
*
52+
* @param handle Service handle from create
53+
* @param audio_data PCM audio data (Int16, 16kHz mono)
54+
* @param audio_size Size of audio data in bytes
55+
* @param options Transcription options
56+
* @param out_result Output: transcription result (text must be strdup'd)
57+
* @param user_data User-provided context
58+
* @return RAC_SUCCESS or error code
59+
*/
60+
typedef rac_result_t (*rac_whisperkit_coreml_stt_transcribe_fn)(rac_handle_t handle,
61+
const void* audio_data,
62+
size_t audio_size,
63+
const rac_stt_options_t* options,
64+
rac_stt_result_t* out_result,
65+
void* user_data);
66+
67+
/**
68+
* Callback to destroy/unload a WhisperKit CoreML service.
69+
*
70+
* @param handle Service handle to destroy
71+
* @param user_data User-provided context
72+
*/
73+
typedef void (*rac_whisperkit_coreml_stt_destroy_fn)(rac_handle_t handle, void* user_data);
74+
75+
/**
76+
* Swift callbacks for WhisperKit CoreML STT operations.
77+
*/
78+
typedef struct rac_whisperkit_coreml_stt_callbacks {
79+
rac_whisperkit_coreml_stt_can_handle_fn can_handle;
80+
rac_whisperkit_coreml_stt_create_fn create;
81+
rac_whisperkit_coreml_stt_transcribe_fn transcribe;
82+
rac_whisperkit_coreml_stt_destroy_fn destroy;
83+
void* user_data;
84+
} rac_whisperkit_coreml_stt_callbacks_t;
85+
86+
// =============================================================================
87+
// CALLBACK REGISTRATION
88+
// =============================================================================
89+
90+
/**
91+
* Sets the Swift callbacks for WhisperKit CoreML STT operations.
92+
* Must be called before rac_backend_whisperkit_coreml_register().
93+
*
94+
* @param callbacks Callback functions (copied internally)
95+
* @return RAC_SUCCESS on success
96+
*/
97+
RAC_API rac_result_t
98+
rac_whisperkit_coreml_stt_set_callbacks(const rac_whisperkit_coreml_stt_callbacks_t* callbacks);
99+
100+
/**
101+
* Gets the current Swift callbacks.
102+
*
103+
* @return Pointer to callbacks, or NULL if not set
104+
*/
105+
RAC_API const rac_whisperkit_coreml_stt_callbacks_t*
106+
rac_whisperkit_coreml_stt_get_callbacks(void);
107+
108+
/**
109+
* Checks if Swift callbacks are registered.
110+
*
111+
* @return RAC_TRUE if callbacks are available
112+
*/
113+
RAC_API rac_bool_t rac_whisperkit_coreml_stt_is_available(void);
114+
115+
// =============================================================================
116+
// BACKEND REGISTRATION
117+
// =============================================================================
118+
119+
/**
120+
* Register the WhisperKit CoreML backend with the module and service registries.
121+
* Swift callbacks must be set via rac_whisperkit_coreml_stt_set_callbacks() first.
122+
*
123+
* @return RAC_SUCCESS on success
124+
*/
125+
RAC_API rac_result_t rac_backend_whisperkit_coreml_register(void);
126+
127+
/**
128+
* Unregister the WhisperKit CoreML backend.
129+
*
130+
* @return RAC_SUCCESS on success
131+
*/
132+
RAC_API rac_result_t rac_backend_whisperkit_coreml_unregister(void);
133+
134+
#ifdef __cplusplus
135+
}
136+
#endif
137+
138+
#endif /* RAC_STT_WHISPERKIT_COREML_H */

‎sdk/runanywhere-commons/src/backends/whisperkit_coreml/CMakeLists.txt‎

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
# =============================================================================
2+
# WhisperKit CoreML Backend - STT via Apple Neural Engine
3+
#
4+
# Unlike other backends, WhisperKit CoreML has no C++ inference engine.
5+
# The C++ side provides callback storage, vtable dispatch, and registration.
6+
# Actual inference happens in Swift via registered callbacks to CoreML.
7+
# =============================================================================
8+
9+
message(STATUS "Configuring WhisperKit CoreML backend...")
10+
11+
# =============================================================================
12+
# WhisperKit CoreML Backend Library
13+
# =============================================================================
14+
15+
set(WHISPERKIT_COREML_BACKEND_SOURCES
16+
rac_stt_whisperkit_coreml.cpp
17+
rac_backend_whisperkit_coreml_register.cpp
18+
)
19+
20+
if(RAC_BUILD_SHARED)
21+
add_library(rac_backend_whisperkit_coreml SHARED ${WHISPERKIT_COREML_BACKEND_SOURCES})
22+
else()
23+
add_library(rac_backend_whisperkit_coreml STATIC ${WHISPERKIT_COREML_BACKEND_SOURCES})
24+
endif()
25+
26+
target_include_directories(rac_backend_whisperkit_coreml PUBLIC
27+
${CMAKE_CURRENT_SOURCE_DIR}
28+
${CMAKE_SOURCE_DIR}/include
29+
)
30+
31+
target_link_libraries(rac_backend_whisperkit_coreml PUBLIC
32+
rac_commons
33+
)
34+
35+
target_compile_features(rac_backend_whisperkit_coreml PUBLIC cxx_std_17)
36+
37+
# =============================================================================
38+
# Summary
39+
# =============================================================================
40+
41+
message(STATUS "WhisperKit CoreML Backend Configuration:")
42+
message(STATUS " Platform: ${RAC_PLATFORM_NAME} (Apple-only)")
43+
message(STATUS " Note: Actual inference runs in Swift via CoreML callbacks")

0 commit comments

Comments
 (0)