Skip to content

Commit 658e729

Browse files
kunal-vaishnavikunal-vaishnavi
authored
Add tool calling and structured outputs to FL SDK v2 (#456)
* Add tool calling and structured outputs to FL C# SDK * Fix JSON parsing and parameter order for unit tests * Fix order of expected arguments * Get FL C# SDK E2E samples via interop and web working * Increase ORT GenAI version to 0.12.1 * Get FL JS SDK E2E samples via interop and web working * Remove changes made for local testing purposes * Add changes suggested during PR review * Separate tool calling examples * Remove unneeded param attribute * Update FL Core versions * Omit passing in FL Core version in C# YAML file for now * Remove useNightly usage from JS build * Remove nightly flag from npm install * Put back useNightly usage in JS build
1 parent 13a3b21 commit 658e729

35 files changed

Lines changed: 1562 additions & 65 deletions

File tree

‎.github/workflows/build-cs-steps.yml‎

Lines changed: 9 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -41,14 +41,17 @@ jobs:
4141
env:
4242
NUGET_AUTH_TOKEN: ${{ secrets.AZURE_DEVOPS_PAT }}
4343

44+
# TODO: once the nightly packaging is fixed, add back the commented out lines with /p:FoundryLocalCoreVersion="*-*"
4445
# /p:FoundryLocalCoreVersion="*-*" to always use nightly version of Foundry Local Core
4546
- name: Restore dependencies
4647
run: |
47-
dotnet restore sdk_v2/cs/src/Microsoft.AI.Foundry.Local.csproj /p:UseWinML=${{ inputs.useWinML }} /p:FoundryLocalCoreVersion="*-*" --configfile sdk_v2/cs/NuGet.config
48+
# dotnet restore sdk_v2/cs/src/Microsoft.AI.Foundry.Local.csproj /p:UseWinML=${{ inputs.useWinML }} /p:FoundryLocalCoreVersion="*-*" --configfile sdk_v2/cs/NuGet.config
49+
dotnet restore sdk_v2/cs/src/Microsoft.AI.Foundry.Local.csproj /p:UseWinML=${{ inputs.useWinML }} --configfile sdk_v2/cs/NuGet.config
4850
4951
- name: Build solution
5052
run: |
51-
dotnet build sdk_v2/cs/src/Microsoft.AI.Foundry.Local.csproj --no-restore --configuration ${{ inputs.buildConfiguration }} /p:UseWinML=${{ inputs.useWinML }} /p:FoundryLocalCoreVersion="*-*"
53+
# dotnet build sdk_v2/cs/src/Microsoft.AI.Foundry.Local.csproj --no-restore --configuration ${{ inputs.buildConfiguration }} /p:UseWinML=${{ inputs.useWinML }} /p:FoundryLocalCoreVersion="*-*"
54+
dotnet build sdk_v2/cs/src/Microsoft.AI.Foundry.Local.csproj --no-restore --configuration ${{ inputs.buildConfiguration }} /p:UseWinML=${{ inputs.useWinML }}
5255
5356
# need to use direct git commands to clone from Azure DevOps instead of actions/checkout
5457
- name: Checkout test-data-shared from Azure DevOps
@@ -82,7 +85,8 @@ jobs:
8285
8386
- name: Run Foundry Local Core tests
8487
run: |
85-
dotnet test sdk_v2/cs/test/FoundryLocal.Tests/Microsoft.AI.Foundry.Local.Tests.csproj --verbosity normal /p:UseWinML=${{ inputs.useWinML }} /p:FoundryLocalCoreVersion="*-*"
88+
# dotnet test sdk_v2/cs/test/FoundryLocal.Tests/Microsoft.AI.Foundry.Local.Tests.csproj --verbosity normal /p:UseWinML=${{ inputs.useWinML }} /p:FoundryLocalCoreVersion="*-*"
89+
dotnet test sdk_v2/cs/test/FoundryLocal.Tests/Microsoft.AI.Foundry.Local.Tests.csproj --verbosity normal /p:UseWinML=${{ inputs.useWinML }}
8690
8791
- name: Pack NuGet package
8892
shell: pwsh
@@ -106,7 +110,8 @@ jobs:
106110
Write-Host "UseWinML: $useWinML"
107111
# Write-Host "FoundryLocalCoreVersion: $coreVersion"
108112
109-
& dotnet pack $projectPath --no-build --configuration $config --output $outputDir /p:PackageVersion=$version /p:UseWinML=$useWinML /p:FoundryLocalCoreVersion="*-*" /p:IncludeSymbols=true /p:SymbolPackageFormat=snupkg --verbosity normal
113+
# & dotnet pack $projectPath --no-build --configuration $config --output $outputDir /p:PackageVersion=$version /p:UseWinML=$useWinML /p:FoundryLocalCoreVersion="*-*" /p:IncludeSymbols=true /p:SymbolPackageFormat=snupkg --verbosity normal
114+
& dotnet pack $projectPath --no-build --configuration $config --output $outputDir /p:PackageVersion=$version /p:UseWinML=$useWinML /p:IncludeSymbols=true /p:SymbolPackageFormat=snupkg --verbosity normal
110115
111116
if ($LASTEXITCODE -ne 0) {
112117
Write-Error "dotnet pack failed with exit code $LASTEXITCODE"

‎.github/workflows/build-js-steps.yml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -88,12 +88,12 @@ jobs:
8888
- name: npm install (WinML)
8989
if: ${{ inputs.useWinML == true }}
9090
working-directory: sdk_v2/js
91-
run: npm install --nightly --winml
91+
run: npm install --winml
9292

9393
- name: npm install (Standard)
9494
if: ${{ inputs.useWinML == false }}
9595
working-directory: sdk_v2/js
96-
run: npm install --nightly
96+
run: npm install
9797

9898
# Verify that installing new packages doesn't strip custom native binary folders
9999
- name: npm install openai (verify persistence)

‎samples/cs/GettingStarted/Directory.Packages.props‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
<Project>
22
<PropertyGroup>
33
<ManagePackageVersionsCentrally>true</ManagePackageVersionsCentrally>
4-
<OnnxRuntimeGenAIVersion>0.11.2</OnnxRuntimeGenAIVersion>
4+
<OnnxRuntimeGenAIVersion>0.12.1</OnnxRuntimeGenAIVersion>
55
<OnnxRuntimeVersion>1.23.2</OnnxRuntimeVersion>
66
</PropertyGroup>
77
<ItemGroup>

‎samples/cs/GettingStarted/README.md‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,8 @@ Both the WinML and cross-platform packages provide the same APIs, so you can eas
1616
- **FoundryLocalWebServer**: A simple console application that shows how to set up a local OpenAI-compliant web server using the Foundry Local SDK.
1717
- **AudioTranscriptionExample**: A simple console application that demonstrates how to use the Foundry Local SDK for audio transcription tasks.
1818
- **ModelManagementExample**: A simple console application that demonstrates how to manage models - such as variant selection and updates - using the Foundry Local SDK.
19+
- **ToolCallingFoundryLocalSdk**: A simple console application that initializes the Foundry Local SDK, downloads a model, loads it and does tool calling with chat completions.
20+
- **ToolCallingFoundryLocalWebServer**: A simple console application that shows how to set up a local OpenAI-compliant web server with tool calling using the Foundry Local SDK.
1921

2022
## Running the samples
2123

‎samples/cs/GettingStarted/cross-platform/FoundrySamplesXPlatform.sln‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,8 @@ VisualStudioVersion = 17.14.36705.20
55
MinimumVisualStudioVersion = 10.0.40219.1
66
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "HelloFoundryLocalSdk", "HelloFoundryLocalSdk\HelloFoundryLocalSdk.csproj", "{785AAE8A-8CD6-4916-B858-29B8A7EF8FF2}"
77
EndProject
8+
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ToolCallingFoundryLocalSdk", "ToolCallingFoundryLocalSdk\ToolCallingFoundryLocalSdk.csproj", "{2F99B88E-BE58-4ED6-A71E-60B6EE955D1B}"
9+
EndProject
810
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "build", "build", "{8EC462FD-D22E-90A8-E5CE-7E832BA40C5D}"
911
ProjectSection(SolutionItems) = preProject
1012
..\Directory.Packages.props = ..\Directory.Packages.props
@@ -14,6 +16,8 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "build", "build", "{8EC462FD
1416
EndProject
1517
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "FoundryLocalWebServer", "FoundryLocalWebServer\FoundryLocalWebServer.csproj", "{D1D6C453-3088-4D8D-B320-24D718601C26}"
1618
EndProject
19+
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ToolCallingFoundryLocalWebServer", "ToolCallingFoundryLocalWebServer\ToolCallingFoundryLocalWebServer.csproj", "{B59762E0-B699-4F80-B2B6-8BC5751A4620}"
20+
EndProject
1721
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "AudioTranscriptionExample", "AudioTranscriptionExample\AudioTranscriptionExample.csproj", "{2FAD8210-8AEB-4063-9C61-57B7AD26772D}"
1822
EndProject
1923
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ModelManagementExample", "ModelManagementExample\ModelManagementExample.csproj", "{AAD0233C-9FDD-46A7-9428-2F72BC76D38E}"

‎samples/cs/GettingStarted/cross-platform/ToolCallingFoundryLocalSdk/ToolCallingFoundryLocalSdk.csproj‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
<Project Sdk="Microsoft.NET.Sdk">
2+
3+
<PropertyGroup>
4+
<OutputType>Exe</OutputType>
5+
<TargetFramework>net9.0</TargetFramework>
6+
<ImplicitUsings>enable</ImplicitUsings>
7+
<Nullable>enable</Nullable>
8+
</PropertyGroup>
9+
10+
<!-- Include the main program -->
11+
<ItemGroup>
12+
<Compile Include="../../src/ToolCallingFoundryLocalSdk/*.cs" />
13+
<Compile Include="../../src/Shared/*.cs" />
14+
</ItemGroup>
15+
16+
<!-- Packages -->
17+
<ItemGroup>
18+
<PackageReference Include="Microsoft.AI.Foundry.Local" />
19+
</ItemGroup>
20+
21+
<!-- ONNX Runtime GPU and CUDA provider (required for Linux)-->
22+
<ItemGroup Condition="'$(RuntimeIdentifier)' == 'linux-x64'">
23+
<PackageReference Include="Microsoft.ML.OnnxRuntime.Gpu" />
24+
<PackageReference Include="Microsoft.ML.OnnxRuntimeGenAI.Cuda" />
25+
</ItemGroup>
26+
27+
28+
<!-- Exclude superfluous ORT and IHV libs -->
29+
<Import Project="../../ExcludeExtraLibs.props" />
30+
31+
</Project>

‎samples/cs/GettingStarted/cross-platform/ToolCallingFoundryLocalWebServer/ToolCallingFoundryLocalWebServer.csproj‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
<Project Sdk="Microsoft.NET.Sdk">
2+
3+
<PropertyGroup>
4+
<OutputType>Exe</OutputType>
5+
<TargetFramework>net9.0</TargetFramework>
6+
<ImplicitUsings>enable</ImplicitUsings>
7+
<Nullable>enable</Nullable>
8+
</PropertyGroup>
9+
10+
<!-- Include the main program -->
11+
<ItemGroup>
12+
<Compile Include="../../src/ToolCallingFoundryLocalWebServer/*.cs" />
13+
<Compile Include="../../src/Shared/*.cs" />
14+
</ItemGroup>
15+
16+
<!-- Packages -->
17+
<ItemGroup>
18+
<PackageReference Include="Microsoft.AI.Foundry.Local" />
19+
<PackageReference Include="OpenAI" />
20+
</ItemGroup>
21+
22+
<!-- ONNX Runtime GPU and CUDA provider (required for Linux)-->
23+
<ItemGroup Condition="'$(RuntimeIdentifier)' == 'linux-x64'">
24+
<PackageReference Include="Microsoft.ML.OnnxRuntime.Gpu" />
25+
<PackageReference Include="Microsoft.ML.OnnxRuntimeGenAI.Cuda" />
26+
</ItemGroup>
27+
28+
29+
<!-- Exclude superfluous ORT and IHV libs -->
30+
<Import Project="../../ExcludeExtraLibs.props" />
31+
32+
33+
</Project>

‎samples/cs/GettingStarted/src/Shared/Utils.cs‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -69,4 +69,10 @@ private static async Task ShowSpinner(string msg, CancellationToken token)
6969

7070
Console.WriteLine($"Done.\n");
7171
}
72+
73+
// Example tool method
74+
internal static int MultiplyNumbers(int first, int second)
75+
{
76+
return first * second;
77+
}
7278
}

‎samples/cs/GettingStarted/src/ToolCallingFoundryLocalSdk/Program.cs‎

Lines changed: 156 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,156 @@
1+
using Microsoft.AI.Foundry.Local;
2+
using Betalgo.Ranul.OpenAI.ObjectModels.RequestModels;
3+
using Betalgo.Ranul.OpenAI.ObjectModels.ResponseModels;
4+
using Betalgo.Ranul.OpenAI.ObjectModels.SharedModels;
5+
using System.Text.Json;
6+
7+
CancellationToken ct = new CancellationToken();
8+
9+
var config = new Configuration
10+
{
11+
AppName = "foundry_local_samples",
12+
LogLevel = Microsoft.AI.Foundry.Local.LogLevel.Information
13+
};
14+
15+
16+
// Initialize the singleton instance.
17+
await FoundryLocalManager.CreateAsync(config, Utils.GetAppLogger());
18+
var mgr = FoundryLocalManager.Instance;
19+
20+
21+
// Ensure that any Execution Provider (EP) downloads run and are completed.
22+
// EP packages include dependencies and may be large.
23+
// Download is only required again if a new version of the EP is released.
24+
// For cross platform builds there is no dynamic EP download and this will return immediately.
25+
await Utils.RunWithSpinner("Registering execution providers", mgr.EnsureEpsDownloadedAsync());
26+
27+
28+
// Get the model catalog
29+
var catalog = await mgr.GetCatalogAsync();
30+
31+
32+
// Get a model using an alias.
33+
var model = await catalog.GetModelAsync("qwen2.5-0.5b") ?? throw new Exception("Model not found");
34+
35+
36+
// Download the model (the method skips download if already cached)
37+
await model.DownloadAsync(progress =>
38+
{
39+
Console.Write($"\rDownloading model: {progress:F2}%");
40+
if (progress >= 100f)
41+
{
42+
Console.WriteLine();
43+
}
44+
});
45+
46+
47+
// Load the model
48+
Console.Write($"Loading model {model.Id}...");
49+
await model.LoadAsync();
50+
Console.WriteLine("done.");
51+
52+
53+
// Get a chat client
54+
var chatClient = await model.GetChatClientAsync();
55+
chatClient.Settings.ToolChoice = ToolChoice.Required; // Force the model to make a tool call
56+
57+
58+
// Prepare messages
59+
List<ChatMessage> messages =
60+
[
61+
new ChatMessage { Role = "system", Content = "You are a helpful AI assistant. If necessary, you can use any provided tools to answer the question." },
62+
new ChatMessage { Role = "user", Content = "What is the answer to 7 multiplied by 6?" }
63+
];
64+
65+
66+
// Prepare tools
67+
List<ToolDefinition> tools =
68+
[
69+
new ToolDefinition
70+
{
71+
Type = "function",
72+
Function = new FunctionDefinition()
73+
{
74+
Name = "multiply_numbers",
75+
Description = "A tool for multiplying two numbers.",
76+
Parameters = new PropertyDefinition()
77+
{
78+
Type = "object",
79+
Properties = new Dictionary<string, PropertyDefinition>()
80+
{
81+
{ "first", new PropertyDefinition() { Type = "integer", Description = "The first number in the operation" } },
82+
{ "second", new PropertyDefinition() { Type = "integer", Description = "The second number in the operation" } }
83+
},
84+
Required = ["first", "second"]
85+
}
86+
}
87+
}
88+
];
89+
90+
91+
// Get a streaming chat completion response
92+
var toolCallResponses = new List<ChatCompletionCreateResponse>();
93+
Console.WriteLine("Chat completion response:");
94+
var streamingResponse = chatClient.CompleteChatStreamingAsync(messages, tools, ct);
95+
await foreach (var chunk in streamingResponse)
96+
{
97+
var content = chunk.Choices[0].Message.Content;
98+
Console.Write(content);
99+
Console.Out.Flush();
100+
101+
if (chunk.Choices[0].FinishReason == "tool_calls")
102+
{
103+
toolCallResponses.Add(chunk);
104+
}
105+
}
106+
Console.WriteLine();
107+
108+
109+
// Invoke tools called and append responses to the chat
110+
foreach (var chunk in toolCallResponses)
111+
{
112+
var call = chunk?.Choices[0].Message.ToolCalls?[0].FunctionCall;
113+
if (call?.Name == "multiply_numbers")
114+
{
115+
var arguments = JsonSerializer.Deserialize<Dictionary<string, int>>(call.Arguments!)!;
116+
var first = arguments["first"];
117+
var second = arguments["second"];
118+
119+
Console.WriteLine($"\nInvoking tool: {call?.Name} with arguments {first} and {second}");
120+
var result = Utils.MultiplyNumbers(first, second);
121+
Console.WriteLine($"Tool response: {result.ToString()}");
122+
123+
var response = new ChatMessage
124+
{
125+
Role = "tool",
126+
Content = result.ToString(),
127+
};
128+
messages.Add(response);
129+
}
130+
}
131+
Console.WriteLine("\nTool calls completed. Prompting model to continue conversation...\n");
132+
133+
134+
// Prompt the model to continue the conversation after the tool call
135+
messages.Add(new ChatMessage { Role = "system", Content = "Respond only with the answer generated by the tool." });
136+
137+
138+
// Set tool calling back to auto so that the model can decide whether to call
139+
// the tool again or continue the conversation based on the new user prompt
140+
chatClient.Settings.ToolChoice = ToolChoice.Auto;
141+
142+
143+
// Run the next turn of the conversation
144+
Console.WriteLine("Chat completion response:");
145+
streamingResponse = chatClient.CompleteChatStreamingAsync(messages, tools, ct);
146+
await foreach (var chunk in streamingResponse)
147+
{
148+
var content = chunk.Choices[0].Message.Content;
149+
Console.Write(content);
150+
Console.Out.Flush();
151+
}
152+
Console.WriteLine();
153+
154+
155+
// Tidy up - unload the model
156+
await model.UnloadAsync();

0 commit comments

Comments
 (0)