Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

fix: Ollama provider #69

Merged
merged 5 commits into from
Dec 2, 2024
Merged

fix: Ollama provider #69

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
1b7e9e6
feat: Add Ollama provider
kaspermarstal Nov 21, 2024
0f7fa25
Merge branch 'main' of https://github.com/getcellm/cellm
kaspermarstal Nov 22, 2024
4478dcf
wip
kaspermarstal Nov 24, 2024
bbca7ba
refactor: ServiceCollectionExtensions, split OpenAI support into Open…
kaspermarstal Nov 30, 2024
ef26ade
feat: Add OpenAiCompatible model provider to appsettings.Local.*.json…
kaspermarstal Dec 2, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions src/Cellm.Tests/Cellm.Tests.csproj
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
<Project Sdk="Microsoft.NET.Sdk">
<Project Sdk="Microsoft.NET.Sdk">

<PropertyGroup>
<TargetFramework>net8.0-windows</TargetFramework>
Expand All @@ -12,8 +12,8 @@
</PropertyGroup>

<ItemGroup>
<PackageReference Include="ExcelDna.Testing" Version="1.8.0" />
<PackageReference Include="xunit" Version="2.9.0" />
<PackageReference Include="ExcelDna.Testing" Version="1.9.0-alpha3" />
<PackageReference Include="xunit" Version="2.9.2" />
</ItemGroup>

</Project>
50 changes: 25 additions & 25 deletions src/Cellm.Tests/packages.lock.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,11 +4,11 @@
"net8.0-windows7.0": {
"ExcelDna.Testing": {
"type": "Direct",
"requested": "[1.8.0, )",
"resolved": "1.8.0",
"contentHash": "nuxqQE7t9O8kASTeGrwdCwJl8p3A1PchmYUNEwmyZHfTviWgaKJpSoR5m2S/c4jr0ejErgXooKKkzPmFb7e22Q==",
"requested": "[1.9.0-alpha3, )",
"resolved": "1.9.0-alpha3",
"contentHash": "1czKqjx7dzU59Ilt0Ewp06aM5wsOgGS+CodRdWumhpyrxz8Hqnp3HCVDsBWhgElEPn0bkcdQFxG+W7xyJBXLkQ==",
"dependencies": {
"ExcelDna.Integration": "[1.8.0]",
"ExcelDna.Integration": "[1.9.0-alpha3]",
"ExcelDna.Interop": "[15.0.1]",
"Microsoft.NET.Test.Sdk": "16.11.0",
"Microsoft.VisualStudio.Interop": "17.1.32210.191",
Expand All @@ -23,19 +23,19 @@
},
"xunit": {
"type": "Direct",
"requested": "[2.9.0, )",
"resolved": "2.9.0",
"contentHash": "PtU3rZ0ThdmdJqTbK7GkgFf6iBaCR6Q0uvJHznID+XEYk2v6O/b7sRxqnbi3B2gRDXxjTqMkVNayzwsqsFUxRw==",
"requested": "[2.9.2, )",
"resolved": "2.9.2",
"contentHash": "7LhFS2N9Z6Xgg8aE5lY95cneYivRMfRI8v+4PATa4S64D5Z/Plkg0qa8dTRHSiGRgVZ/CL2gEfJDE5AUhOX+2Q==",
"dependencies": {
"xunit.analyzers": "1.15.0",
"xunit.assert": "2.9.0",
"xunit.core": "[2.9.0]"
"xunit.analyzers": "1.16.0",
"xunit.assert": "2.9.2",
"xunit.core": "[2.9.2]"
}
},
"ExcelDna.Integration": {
"type": "Transitive",
"resolved": "1.8.0",
"contentHash": "Z7UYuY291cTxc8lEESlUT73iI28yZ9LeJnvhPIRpZHBlrjbCN8lnEWvSmUqmUn1lBNGk6p2n0GlPuBhopJuksA=="
"resolved": "1.9.0-alpha3",
"contentHash": "vQZP+bXCXkiIx4bkntd2iWnMDS57ECcul2XFQvMSvgcTy6+2q81xI2Sp6FGXr9tU/t7eHfpG7vVv/7TZKOLnxA=="
},
"ExcelDna.Interop": {
"type": "Transitive",
Expand Down Expand Up @@ -810,37 +810,37 @@
},
"xunit.analyzers": {
"type": "Transitive",
"resolved": "1.15.0",
"contentHash": "s+M8K/Rtlgr6CmD7AYQKrNTvT5sh0l0ZKDoZ3Z/ExhlIwfV9mGAMR4f7KqIB7SSK7ZOhqDTgTUMYPmKfmvWUWQ=="
"resolved": "1.16.0",
"contentHash": "hptYM7vGr46GUIgZt21YHO4rfuBAQS2eINbFo16CV/Dqq+24Tp+P5gDCACu1AbFfW4Sp/WRfDPSK8fmUUb8s0Q=="
},
"xunit.assert": {
"type": "Transitive",
"resolved": "2.9.0",
"contentHash": "Z/1pyia//860wEYTKn6Q5dmgikJdRjgE4t5AoxJkK8oTmidzPLEPG574kmm7LFkMLbH6Frwmgb750kcyR+hwoA=="
"resolved": "2.9.2",
"contentHash": "QkNBAQG4pa66cholm28AxijBjrmki98/vsEh4Sx5iplzotvPgpiotcxqJQMRC8d7RV7nIT8ozh97957hDnZwsQ=="
},
"xunit.core": {
"type": "Transitive",
"resolved": "2.9.0",
"contentHash": "uRaop9tZsZMCaUS4AfbSPGYHtvywWnm8XXFNUqII7ShWyDBgdchY6gyDNgO4AK1Lv/1NNW61Zq63CsDV6oH6Jg==",
"resolved": "2.9.2",
"contentHash": "O6RrNSdmZ0xgEn5kT927PNwog5vxTtKrWMihhhrT0Sg9jQ7iBDciYOwzBgP2krBEk5/GBXI18R1lKvmnxGcb4w==",
"dependencies": {
"xunit.extensibility.core": "[2.9.0]",
"xunit.extensibility.execution": "[2.9.0]"
"xunit.extensibility.core": "[2.9.2]",
"xunit.extensibility.execution": "[2.9.2]"
}
},
"xunit.extensibility.core": {
"type": "Transitive",
"resolved": "2.9.0",
"contentHash": "zjDEUSxsr6UNij4gIwCgMqQox+oLDPRZ+mubwWLci+SssPBFQD1xeRR4SvgBuXqbE0QXCJ/STVTp+lxiB5NLVA==",
"resolved": "2.9.2",
"contentHash": "Ol+KlBJz1x8BrdnhN2DeOuLrr1I/cTwtHCggL9BvYqFuVd/TUSzxNT5O0NxCIXth30bsKxgMfdqLTcORtM52yQ==",
"dependencies": {
"xunit.abstractions": "2.0.3"
}
},
"xunit.extensibility.execution": {
"type": "Transitive",
"resolved": "2.9.0",
"contentHash": "5ZTQZvmPLlBw6QzCOwM0KnMsZw6eGjbmC176QHZlcbQoMhGIeGcYzYwn5w9yXxf+4phtplMuVqTpTbFDQh2bqQ==",
"resolved": "2.9.2",
"contentHash": "rKMpq4GsIUIJibXuZoZ8lYp5EpROlnYaRpwu9Zr0sRZXE7JqJfEEbCsUriZqB+ByXCLFBJyjkTRULMdC+U566g==",
"dependencies": {
"xunit.extensibility.core": "[2.9.0]"
"xunit.extensibility.core": "[2.9.2]"
}
},
"xunit.runner.visualstudio": {
Expand Down
29 changes: 17 additions & 12 deletions src/Cellm/Cellm.csproj
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,30 +18,35 @@
</ItemGroup>

<ItemGroup>
<PackageReference Include="ExcelDna.Addin" Version="1.8.0" />
<PackageReference Include="ExcelDna.Addin" Version="1.9.0-alpha3" />
<PackageReference Include="ExcelDna.Interop" Version="15.0.1" />
<PackageReference Include="JsonPatch.Net" Version="3.1.1" />
<PackageReference Include="JsonSchema.Net.Generation" Version="4.5.1" />
<PackageReference Include="MediatR" Version="12.4.1" />
<PackageReference Include="Microsoft.Extensions.AI" Version="9.0.0-preview.9.24556.5" />
<PackageReference Include="Microsoft.Extensions.AI.Abstractions" Version="9.0.0-preview.9.24556.5" />
<PackageReference Include="Microsoft.Extensions.AI.Ollama" Version="9.0.0-preview.9.24556.5" />
<PackageReference Include="Microsoft.Extensions.AI.OpenAI" Version="9.0.0-preview.9.24556.5" />
<PackageReference Include="Microsoft.Extensions.AI" Version="9.0.1-preview.1.24570.5" />
<PackageReference Include="Microsoft.Extensions.AI.Abstractions" Version="9.0.1-preview.1.24570.5" />
<PackageReference Include="Microsoft.Extensions.AI.Ollama" Version="9.0.1-preview.1.24570.5" />
<PackageReference Include="Microsoft.Extensions.AI.OpenAI" Version="9.0.1-preview.1.24570.5" />
<PackageReference Include="Microsoft.Extensions.Caching.Hybrid" Version="9.0.0-preview.9.24556.5" />
<PackageReference Include="Microsoft.Extensions.Caching.Memory" Version="9.0.0" />
<PackageReference Include="Microsoft.Extensions.Configuration" Version="9.0.0" />
<PackageReference Include="Microsoft.Extensions.Configuration.Json" Version="8.0.1" />
<PackageReference Include="Microsoft.Extensions.Configuration.Json" Version="9.0.0" />
<PackageReference Include="Microsoft.Extensions.DependencyInjection" Version="9.0.0" />
<PackageReference Include="Microsoft.Extensions.FileSystemGlobbing" Version="8.0.0" />
<PackageReference Include="Microsoft.Extensions.Http" Version="8.0.0" />
<PackageReference Include="Microsoft.Extensions.Http.Resilience" Version="8.8.0" />
<PackageReference Include="Microsoft.Extensions.FileSystemGlobbing" Version="9.0.0" />
<PackageReference Include="Microsoft.Extensions.Http" Version="9.0.0" />
<PackageReference Include="Microsoft.Extensions.Http.Resilience" Version="9.0.0" />
<PackageReference Include="Microsoft.Extensions.Logging" Version="9.0.0" />
<PackageReference Include="Microsoft.Extensions.Logging.Console" Version="9.0.0" />
<PackageReference Include="Microsoft.Extensions.Logging.Debug" Version="8.0.0" />
<PackageReference Include="Microsoft.Extensions.Logging.Debug" Version="9.0.0" />
<PackageReference Include="Microsoft.Extensions.Options" Version="9.0.0" />
<PackageReference Include="Microsoft.Extensions.Options.ConfigurationExtensions" Version="9.0.0" />
<PackageReference Include="PdfPig" Version="0.1.9" />
<PackageReference Include="Sentry.Extensions.Logging" Version="4.12.1" />
<PackageReference Include="Sentry.Profiling" Version="4.12.1" />
<PackageReference Include="Sentry.Extensions.Logging" Version="4.13.0" />
<PackageReference Include="Sentry.Profiling" Version="4.13.0" />
<PackageReference Include="System.Diagnostics.DiagnosticSource" Version="9.0.0">
<ExcludeAssets>runtime</ExcludeAssets>
</PackageReference>
<PackageReference Include="System.Net.Http.Json" Version="9.0.0" />
<Content Include="appsettings.json">
<CopyToOutputDirectory>Always</CopyToOutputDirectory>
</Content>
Expand Down
2 changes: 1 addition & 1 deletion src/Cellm/Models/Anthropic/AnthropicRequestHandler.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,7 @@ public string Serialize(AnthropicRequest request)
var requestBody = new AnthropicRequestBody
{
System = request.Prompt.Messages.Where(x => x.Role == ChatRole.System).First().Text,
Messages = request.Prompt.Messages.Select(x => new AnthropicMessage { Content = x.Text, Role = x.Role.ToString().ToLower() }).ToList(),
Messages = request.Prompt.Messages.Where(x => x.Role != ChatRole.System).Select(x => new AnthropicMessage { Content = x.Text, Role = x.Role.ToString().ToLower() }).ToList(),
Model = request.Prompt.Options.ModelId ?? _anthropicConfiguration.DefaultModel,
MaxTokens = _cellmConfiguration.MaxOutputTokens,
Temperature = request.Prompt.Options.Temperature ?? _cellmConfiguration.DefaultTemperature,
Expand Down
4 changes: 2 additions & 2 deletions src/Cellm/Models/Client.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,8 +38,8 @@ public async Task<Prompt> Send(Prompt prompt, string? provider, Uri? baseAddress
{
Providers.Anthropic => await _sender.Send(new AnthropicRequest(prompt, provider, baseAddress)),
Providers.Llamafile => await _sender.Send(new LlamafileRequest(prompt)),
Providers.Ollama => await _sender.Send(new OllamaRequest(prompt, provider, baseAddress)),
Providers.OpenAi => await _sender.Send(new OpenAiRequest(prompt, provider, baseAddress)),
Providers.Ollama => await _sender.Send(new OllamaRequest(prompt)),
Providers.OpenAi => await _sender.Send(new OpenAiRequest(prompt)),
_ => throw new InvalidOperationException($"Provider {parsedProvider} is defined but not implemented")
};

Expand Down
51 changes: 33 additions & 18 deletions src/Cellm/Models/Llamafile/LlamafileRequestHandler.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,9 @@
using Cellm.AddIn;
using Cellm.AddIn.Exceptions;
using Cellm.Models.Local;
using Cellm.Models.OpenAi;
using Cellm.Models.OpenAiCompatible;
using MediatR;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;

namespace Cellm.Models.Llamafile;
Expand All @@ -22,35 +23,46 @@ private record Llamafile(string ModelPath, Uri BaseAddress, Process Process);
private readonly ISender _sender;
private readonly HttpClient _httpClient;
private readonly LocalUtilities _localUtilities;
private readonly ILogger<LlamafileRequestHandler> _logger;

public LlamafileRequestHandler(IOptions<CellmConfiguration> cellmConfiguration,
IOptions<LlamafileConfiguration> llamafileConfiguration,
ISender sender,
HttpClient httpClient,
LocalUtilities localUtilities,
ProcessManager processManager)
ProcessManager processManager,
ILogger<LlamafileRequestHandler> logger)
{
_cellmConfiguration = cellmConfiguration.Value;
_llamafileConfiguration = llamafileConfiguration.Value;
_sender = sender;
_httpClient = httpClient;
_localUtilities = localUtilities;
_processManager = processManager;
_logger = logger;

_llamafileExePath = new AsyncLazy<string>(async () =>
{
var llamafileName = Path.GetFileName(_llamafileConfiguration.LlamafileUrl.Segments.Last());
return await _localUtilities.DownloadFile(_llamafileConfiguration.LlamafileUrl, $"{llamafileName}.exe");
return await _localUtilities.DownloadFileIfNotExists(_llamafileConfiguration.LlamafileUrl, _localUtilities.CreateCellmFilePath(CreateModelFileName($"{llamafileName}.exe"), "Llamafile"));
});

_llamafiles = _llamafileConfiguration.Models.ToDictionary(x => x.Key, x => new AsyncLazy<Llamafile>(async () =>
{
// Download Llamafile
var exePath = await _llamafileExePath;

// Download model
var modelPath = await _localUtilities.DownloadFile(x.Value, _localUtilities.CreateCellmFilePath(CreateModelFileName(x.Key)));
var modelPath = await _localUtilities.DownloadFileIfNotExists(x.Value, _localUtilities.CreateCellmFilePath(CreateModelFileName(x.Key), "Llamafile"));

// Start server
var baseAddress = new UriBuilder("http", "localhost", _localUtilities.FindPort()).Uri;
var process = await StartProcess(modelPath, baseAddress);
var baseAddress = new UriBuilder(
_llamafileConfiguration.BaseAddress.Scheme,
_llamafileConfiguration.BaseAddress.Host,
_localUtilities.FindPort(),
_llamafileConfiguration.BaseAddress.AbsolutePath).Uri;

var process = await StartProcess(exePath, modelPath, baseAddress);

return new Llamafile(modelPath, baseAddress, process);
}));
Expand All @@ -61,20 +73,23 @@ public async Task<LlamafileResponse> Handle(LlamafileRequest request, Cancellati
// Start server on first call
var llamafile = await _llamafiles[request.Prompt.Options.ModelId ?? _llamafileConfiguration.DefaultModel];

var openAiResponse = await _sender.Send(new OpenAiRequest(request.Prompt, nameof(Llamafile), llamafile.BaseAddress), cancellationToken);
var openAiResponse = await _sender.Send(new OpenAiCompatibleRequest(request.Prompt, nameof(Llamafile), llamafile.BaseAddress), cancellationToken);

return new LlamafileResponse(openAiResponse.Prompt);
}

private async Task<Process> StartProcess(string modelPath, Uri baseAddress)
private async Task<Process> StartProcess(string exePath, string modelPath, Uri baseAddress)
{
var processStartInfo = new ProcessStartInfo(await _llamafileExePath);
var processStartInfo = new ProcessStartInfo(exePath);

processStartInfo.Arguments += $"--server ";
processStartInfo.Arguments += "--nobrowser ";
processStartInfo.Arguments += $"-m {modelPath} ";
processStartInfo.Arguments += $"--host {baseAddress.Host} ";
processStartInfo.Arguments += $"--port {baseAddress.Port} ";
processStartInfo.ArgumentList.Add("--server");
processStartInfo.ArgumentList.Add("--nobrowser");
processStartInfo.ArgumentList.Add("-m");
processStartInfo.ArgumentList.Add(modelPath);
processStartInfo.ArgumentList.Add("--host");
processStartInfo.ArgumentList.Add(baseAddress.Host);
processStartInfo.ArgumentList.Add("--port");
processStartInfo.ArgumentList.Add(baseAddress.Port.ToString());

if (_llamafileConfiguration.Gpu)
{
Expand All @@ -94,16 +109,16 @@ private async Task<Process> StartProcess(string modelPath, Uri baseAddress)
{
if (!string.IsNullOrEmpty(e.Data))
{
Debug.WriteLine(e.Data);
_logger.LogDebug(e.Data);
}
};

process.BeginOutputReadLine();
process.BeginErrorReadLine();
}

var address = new Uri(baseAddress, "health");
await _localUtilities.WaitForServer(address, process);
var uriBuilder = new UriBuilder(baseAddress.Scheme, baseAddress.Host, baseAddress.Port, "/health");
await _localUtilities.WaitForServer(uriBuilder.Uri, process);

// Kill Llamafile when Excel exits or dies
_processManager.AssignProcessToExcel(process);
Expand All @@ -113,7 +128,7 @@ private async Task<Process> StartProcess(string modelPath, Uri baseAddress)

private static string CreateModelFileName(string modelName)
{
return $"Llamafile-model-{modelName}";
return $"Llamafile-{modelName}";
}
}

Loading