From 1b7e9e6f480619e1b92c62d1cb15048e13eb64f0 Mon Sep 17 00:00:00 2001 From: Kasper Marstal Date: Thu, 21 Nov 2024 23:19:00 +0100 Subject: [PATCH] feat: Add Ollama provider --- README.md | 2 +- src/Cellm/Models/Client.cs | 2 + .../Llamafile/LlamafileRequestHandler.cs | 143 ++--------------- src/Cellm/Models/Local/LocalUtilities.cs | 150 ++++++++++++++++++ .../ProcessManager.cs} | 4 +- .../Models/Ollama/OllamaConfiguration.cs | 19 +++ src/Cellm/Models/Ollama/OllamaRequest.cs | 5 + .../Models/Ollama/OllamaRequestHandler.cs | 144 +++++++++++++++++ src/Cellm/Models/Ollama/OllamaResponse.cs | 5 + src/Cellm/Models/Providers.cs | 1 + src/Cellm/Services/ServiceLocator.cs | 28 +++- src/Cellm/appsettings.Local.Ollama.json | 5 +- src/Cellm/appsettings.json | 7 +- 13 files changed, 370 insertions(+), 145 deletions(-) create mode 100644 src/Cellm/Models/Local/LocalUtilities.cs rename src/Cellm/Models/{Llamafile/LLamafileProcessManager.cs => Local/ProcessManager.cs} (97%) create mode 100644 src/Cellm/Models/Ollama/OllamaConfiguration.cs create mode 100644 src/Cellm/Models/Ollama/OllamaRequest.cs create mode 100644 src/Cellm/Models/Ollama/OllamaRequestHandler.cs create mode 100644 src/Cellm/Models/Ollama/OllamaResponse.cs diff --git a/README.md b/README.md index 6e3162e..6bfb8e6 100644 --- a/README.md +++ b/README.md @@ -204,7 +204,7 @@ To get started, we recommend using Ollama with the Gemma 2 2B model: 1. Rename `appsettings.Ollama.json` to `appsettings.Local.json`, 2. Build and install Cellm. -3. Run the following command in the docker directory: +3. Run the following command in the `docker/` directory: ```cmd docker compose -f docker-compose.Ollama.yml up --detach docker compose -f docker-compose.Ollama.yml exec backend ollama pull gemma2:2b diff --git a/src/Cellm/Models/Client.cs b/src/Cellm/Models/Client.cs index 5fee856..0fb7482 100644 --- a/src/Cellm/Models/Client.cs +++ b/src/Cellm/Models/Client.cs @@ -3,6 +3,7 @@ using Cellm.AddIn.Exceptions; using Cellm.Models.Anthropic; using Cellm.Models.Llamafile; +using Cellm.Models.Ollama; using Cellm.Models.OpenAi; using Cellm.Prompts; using MediatR; @@ -37,6 +38,7 @@ public async Task Send(Prompt prompt, string? provider, Uri? baseAddress { Providers.Anthropic => await _sender.Send(new AnthropicRequest(prompt, provider, baseAddress)), Providers.Llamafile => await _sender.Send(new LlamafileRequest(prompt)), + Providers.Ollama => await _sender.Send(new OllamaRequest(prompt, provider, baseAddress)), Providers.OpenAi => await _sender.Send(new OpenAiRequest(prompt, provider, baseAddress)), _ => throw new InvalidOperationException($"Provider {parsedProvider} is defined but not implemented") }; diff --git a/src/Cellm/Models/Llamafile/LlamafileRequestHandler.cs b/src/Cellm/Models/Llamafile/LlamafileRequestHandler.cs index f52aecc..dfacaff 100644 --- a/src/Cellm/Models/Llamafile/LlamafileRequestHandler.cs +++ b/src/Cellm/Models/Llamafile/LlamafileRequestHandler.cs @@ -1,7 +1,7 @@ using System.Diagnostics; -using System.Net.NetworkInformation; using Cellm.AddIn; using Cellm.AddIn.Exceptions; +using Cellm.Models.Local; using Cellm.Models.OpenAi; using MediatR; using Microsoft.Extensions.Options; @@ -14,41 +14,42 @@ private record Llamafile(string ModelPath, Uri BaseAddress, Process Process); private readonly AsyncLazy _llamafileExePath; private readonly Dictionary> _llamafiles; - private readonly LLamafileProcessManager _llamafileProcessManager; + private readonly ProcessManager _processManager; private readonly CellmConfiguration _cellmConfiguration; private readonly LlamafileConfiguration _llamafileConfiguration; - private readonly OpenAiConfiguration _openAiConfiguration; private readonly ISender _sender; private readonly HttpClient _httpClient; + private readonly LocalUtilities _localUtilities; public LlamafileRequestHandler(IOptions cellmConfiguration, IOptions llamafileConfiguration, - IOptions openAiConfiguration, ISender sender, HttpClient httpClient, - LLamafileProcessManager llamafileProcessManager) + LocalUtilities localUtilities, + ProcessManager processManager) { _cellmConfiguration = cellmConfiguration.Value; _llamafileConfiguration = llamafileConfiguration.Value; - _openAiConfiguration = openAiConfiguration.Value; _sender = sender; _httpClient = httpClient; - _llamafileProcessManager = llamafileProcessManager; + _localUtilities = localUtilities; + _processManager = processManager; _llamafileExePath = new AsyncLazy(async () => { - return await DownloadFile(_llamafileConfiguration.LlamafileUrl, $"{nameof(Llamafile)}.exe"); + var llamafileName = Path.GetFileName(_llamafileConfiguration.LlamafileUrl.Segments.Last()); + return await _localUtilities.DownloadFile(_llamafileConfiguration.LlamafileUrl, $"{llamafileName}.exe"); }); _llamafiles = _llamafileConfiguration.Models.ToDictionary(x => x.Key, x => new AsyncLazy(async () => { // Download model - var modelPath = await DownloadFile(x.Value, CreateFilePath(CreateModelFileName(x.Key))); + var modelPath = await _localUtilities.DownloadFile(x.Value, _localUtilities.CreateCellmFilePath(CreateModelFileName(x.Key))); - // Run Llamafile - var baseAddress = CreateBaseAddress(); + // Start server + var baseAddress = new UriBuilder("http", "localhost", _localUtilities.FindPort()).Uri; var process = await StartProcess(modelPath, baseAddress); return new Llamafile(modelPath, baseAddress, process); @@ -101,130 +102,18 @@ private async Task StartProcess(string modelPath, Uri baseAddress) process.BeginErrorReadLine(); } - await WaitForLlamafile(baseAddress, process); + var address = new Uri(baseAddress, "health"); + await _localUtilities.WaitForServer(address, process); - // Kill the process when Excel exits or dies - _llamafileProcessManager.AssignProcessToExcel(process); + // Kill Llamafile when Excel exits or dies + _processManager.AssignProcessToExcel(process); return process; } - private async Task DownloadFile(Uri uri, string filePath) - { - if (File.Exists(filePath)) - { - return filePath; - } - - var filePathPart = $"{filePath}.part"; - - if (File.Exists(filePathPart)) - { - File.Delete(filePathPart); - } - - var response = await _httpClient.GetAsync(uri, HttpCompletionOption.ResponseHeadersRead); - response.EnsureSuccessStatusCode(); - - using (var fileStream = File.Create(filePathPart)) - using (var httpStream = await response.Content.ReadAsStreamAsync()) - { - - await httpStream.CopyToAsync(fileStream); - } - - File.Move(filePathPart, filePath); - - return filePath; - } - - private async Task WaitForLlamafile(Uri baseAddress, Process process) - { - var startTime = DateTime.UtcNow; - - // Wait max 30 seconds to load model - while ((DateTime.UtcNow - startTime).TotalSeconds < 30) - { - if (process.HasExited) - { - throw new CellmException($"Failed to run Llamafile, process exited. Exit code: {process.ExitCode}"); - } - - try - { - var cancellationTokenSource = new CancellationTokenSource(TimeSpan.FromSeconds(1)); - var response = await _httpClient.GetAsync(new Uri(baseAddress, "health"), cancellationTokenSource.Token); - if (response.StatusCode == System.Net.HttpStatusCode.OK) - { - // Server is ready - return; - } - } - catch (HttpRequestException) - { - } - catch (TaskCanceledException) - { - } - - // Wait before next attempt - await Task.Delay(500); - } - - process.Kill(); - - throw new CellmException("Failed to run Llamafile, timeout waiting for Llamafile server to start"); - } - - string CreateFilePath(string fileName) - { - var filePath = Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.ApplicationData), nameof(Cellm), fileName); - Directory.CreateDirectory(Path.GetDirectoryName(filePath) ?? throw new CellmException("Failed to create Llamafile folder")); - return filePath; - } - private static string CreateModelFileName(string modelName) { return $"Llamafile-model-{modelName}"; } - - private Uri CreateBaseAddress() - { - var uriBuilder = new UriBuilder(_llamafileConfiguration.BaseAddress) - { - Port = GetFirstUnusedPort() - }; - - return uriBuilder.Uri; - } - - private static int GetFirstUnusedPort(ushort min = 49152, ushort max = 65535) - { - if (max < min) - { - throw new ArgumentException("Max port must be larger than min port."); - } - - var ipProperties = IPGlobalProperties.GetIPGlobalProperties(); - - var activePorts = ipProperties.GetActiveTcpConnections() - .Where(connection => connection.State != TcpState.Closed) - .Select(connection => connection.LocalEndPoint) - .Concat(ipProperties.GetActiveTcpListeners()) - .Concat(ipProperties.GetActiveUdpListeners()) - .Select(endpoint => endpoint.Port) - .ToArray(); - - var firstInactivePort = Enumerable.Range(min, max) - .Where(port => !activePorts.Contains(port)) - .FirstOrDefault(); - - if (firstInactivePort == default) - { - throw new CellmException($"All local TCP ports between {min} and {max} are currently in use."); - } - - return firstInactivePort; - } } diff --git a/src/Cellm/Models/Local/LocalUtilities.cs b/src/Cellm/Models/Local/LocalUtilities.cs new file mode 100644 index 0000000..8194932 --- /dev/null +++ b/src/Cellm/Models/Local/LocalUtilities.cs @@ -0,0 +1,150 @@ +using System.Diagnostics; +using System.IO.Compression; +using System.Net.NetworkInformation; +using Cellm.AddIn.Exceptions; +using Microsoft.Office.Interop.Excel; + +namespace Cellm.Models.Local; + +internal class LocalUtilities(HttpClient httpClient) +{ + public async Task DownloadFile(Uri uri, string filePath) + { + if (File.Exists(filePath)) + { + return filePath; + } + + var filePathPart = $"{filePath}.part"; + + if (File.Exists(filePathPart)) + { + File.Delete(filePathPart); + } + + var response = await httpClient.GetAsync(uri, HttpCompletionOption.ResponseHeadersRead); + response.EnsureSuccessStatusCode(); + + using (var fileStream = File.Create(filePathPart)) + using (var httpStream = await response.Content.ReadAsStreamAsync()) + { + + await httpStream.CopyToAsync(fileStream); + } + + File.Move(filePathPart, filePath); + + return filePath; + } + + public async Task WaitForServer(Uri endpoint, Process process) + { + var startTime = DateTime.UtcNow; + + // Wait max 30 seconds to load model + while ((DateTime.UtcNow - startTime).TotalSeconds < 30) + { + if (process.HasExited) + { + throw new CellmException($"Failed to run Llamafile, process exited. Exit code: {process.ExitCode}"); + } + + try + { + var cancellationTokenSource = new CancellationTokenSource(TimeSpan.FromSeconds(1)); + var response = await httpClient.GetAsync(endpoint, cancellationTokenSource.Token); + if (response.StatusCode == System.Net.HttpStatusCode.OK) + { + // Server is ready + return; + } + } + catch (HttpRequestException) + { + } + catch (TaskCanceledException) + { + } + + // Wait before next attempt + await Task.Delay(500); + } + + process.Kill(); + + throw new CellmException("Failed to run Llamafile, timeout waiting for Llamafile server to start"); + } + + public string CreateCellmDirectory(params string[] subFolders) + { + var folderPath = Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.ApplicationData), nameof(Cellm)); + + if (subFolders.Length > 0) + { + folderPath = Path.Combine(subFolders.Prepend(folderPath).ToArray()); + } + + Directory.CreateDirectory(folderPath); + return folderPath; + } + + public string CreateCellmFilePath(string fileName) + { + return Path.Combine(CreateCellmDirectory(), fileName); + } + + public int FindPort(ushort min = 49152, ushort max = 65535) + { + if (max < min) + { + throw new ArgumentException("Max port must be larger than min port."); + } + + var ipProperties = IPGlobalProperties.GetIPGlobalProperties(); + + var activePorts = ipProperties.GetActiveTcpConnections() + .Where(connection => connection.State != TcpState.Closed) + .Select(connection => connection.LocalEndPoint) + .Concat(ipProperties.GetActiveTcpListeners()) + .Concat(ipProperties.GetActiveUdpListeners()) + .Select(endpoint => endpoint.Port) + .ToArray(); + + var firstInactivePort = Enumerable.Range(min, max) + .Where(port => !activePorts.Contains(port)) + .FirstOrDefault(); + + if (firstInactivePort == default) + { + throw new CellmException($"All local TCP ports between {min} and {max} are currently in use."); + } + + return firstInactivePort; + } + + public string ExtractFile(string zipFilePath, string targetDirectory) + { + using (ZipArchive archive = ZipFile.OpenRead(zipFilePath)) + { + foreach (ZipArchiveEntry entry in archive.Entries) + { + string destinationPath = Path.Combine(targetDirectory, entry.FullName); + + if (!File.Exists(destinationPath)) + { + ZipFile.ExtractToDirectory(zipFilePath, targetDirectory); + return targetDirectory; + } + + var fileInfo = new FileInfo(destinationPath); + if (fileInfo.Length != entry.Length) + { + ZipFile.ExtractToDirectory(zipFilePath, targetDirectory); + return targetDirectory; + } + } + } + + return targetDirectory; + } +} diff --git a/src/Cellm/Models/Llamafile/LLamafileProcessManager.cs b/src/Cellm/Models/Local/ProcessManager.cs similarity index 97% rename from src/Cellm/Models/Llamafile/LLamafileProcessManager.cs rename to src/Cellm/Models/Local/ProcessManager.cs index 5bc584a..da2d4c9 100644 --- a/src/Cellm/Models/Llamafile/LLamafileProcessManager.cs +++ b/src/Cellm/Models/Local/ProcessManager.cs @@ -1,7 +1,7 @@ using System.Diagnostics; using System.Runtime.InteropServices; -public class LLamafileProcessManager +public class ProcessManager { [DllImport("kernel32.dll", CharSet = CharSet.Unicode)] static extern IntPtr CreateJobObject(IntPtr a, string lpName); @@ -61,7 +61,7 @@ enum JobObjectInfoType private IntPtr _jobObject; - public LLamafileProcessManager() + public ProcessManager() { _jobObject = CreateJobObject(IntPtr.Zero, string.Empty); diff --git a/src/Cellm/Models/Ollama/OllamaConfiguration.cs b/src/Cellm/Models/Ollama/OllamaConfiguration.cs new file mode 100644 index 0000000..7183225 --- /dev/null +++ b/src/Cellm/Models/Ollama/OllamaConfiguration.cs @@ -0,0 +1,19 @@ +using Cellm.Services.Configuration; + +namespace Cellm.Models.Ollama; + +internal class OllamaConfiguration : IProviderConfiguration +{ + public Uri OllamaUri { get; init; } + + public Uri BaseAddress { get; init; } + + public string DefaultModel { get; init; } + + public OllamaConfiguration() + { + OllamaUri = default!; + BaseAddress = default!; + DefaultModel = default!; + } +} \ No newline at end of file diff --git a/src/Cellm/Models/Ollama/OllamaRequest.cs b/src/Cellm/Models/Ollama/OllamaRequest.cs new file mode 100644 index 0000000..142d2ea --- /dev/null +++ b/src/Cellm/Models/Ollama/OllamaRequest.cs @@ -0,0 +1,5 @@ +using Cellm.Prompts; + +namespace Cellm.Models.Ollama; + +internal record OllamaRequest(Prompt Prompt, string? Provider, Uri? BaseAddress) : IModelRequest; diff --git a/src/Cellm/Models/Ollama/OllamaRequestHandler.cs b/src/Cellm/Models/Ollama/OllamaRequestHandler.cs new file mode 100644 index 0000000..5e70725 --- /dev/null +++ b/src/Cellm/Models/Ollama/OllamaRequestHandler.cs @@ -0,0 +1,144 @@ +using System.Diagnostics; +using System.IO.Compression; +using System.Net.Http.Json; +using System.Runtime.InteropServices; +using System.Text; +using System.Text.Json; +using Cellm.AddIn; +using Cellm.AddIn.Exceptions; +using Cellm.Models.Llamafile; +using Cellm.Models.Local; +using Cellm.Prompts; +using Microsoft.Extensions.AI; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using Microsoft.Office.Interop.Excel; + +namespace Cellm.Models.Ollama; + +internal class OllamaRequestHandler : IModelRequestHandler +{ + private record Ollama(Uri BaseAddress, Process Process); + record Model(string Name); + + private readonly CellmConfiguration _cellmConfiguration; + private readonly OllamaConfiguration _ollamaConfiguration; + private readonly HttpClient _httpClient; + private readonly LocalUtilities _localUtilities; + private readonly ProcessManager _processManager; + private readonly ILogger _logger; + + private readonly AsyncLazy _ollamaExePath; + private readonly AsyncLazy _ollama; + + public OllamaRequestHandler( + IOptions cellmConfiguration, + IOptions ollamaConfiguration, + HttpClient httpClient, + LocalUtilities localUtilities, + ProcessManager processManager, + ILogger logger) + { + _cellmConfiguration = cellmConfiguration.Value; + _ollamaConfiguration = ollamaConfiguration.Value; + _httpClient = httpClient; + _localUtilities = localUtilities; + _processManager = processManager; + _logger = logger; + + _ollamaExePath = new AsyncLazy(async () => + { + var zipFileName = string.Join("-", _ollamaConfiguration.OllamaUri.Segments.TakeLast(2)); + var zipFilePath = _localUtilities.CreateCellmFilePath(zipFileName); + + await _localUtilities.DownloadFile(_ollamaConfiguration.OllamaUri, zipFilePath); + var ollamaPath = _localUtilities.ExtractFile(zipFilePath, _localUtilities.CreateCellmDirectory(nameof(Ollama), Path.GetFileNameWithoutExtension(zipFileName))); + return Path.Combine(ollamaPath, "ollama.exe"); + }); + + _ollama = new AsyncLazy(async () => + { + var baseAddress = new UriBuilder("http", "localhost", _localUtilities.FindPort()).Uri; + var process = await StartProcess(baseAddress); + + return new Ollama(baseAddress, process); + }); + } + + public async Task Handle(OllamaRequest request, CancellationToken cancellationToken) + { + // Start server on first call + _ = await _ollama; + + var modelId = request.Prompt.Options.ModelId ?? _ollamaConfiguration.DefaultModel; + + const string path = "/v1/chat/completions"; + var address = request.BaseAddress is null ? new Uri(path, UriKind.Relative) : new Uri(request.BaseAddress, path); + + // Must instantiate manually because address can be set/changed only at instantiation + var chatClient = await GetChatClient(address, modelId); + var chatCompletion = await chatClient.CompleteAsync(request.Prompt.Messages, request.Prompt.Options, cancellationToken); + + var prompt = new PromptBuilder(request.Prompt) + .AddMessage(chatCompletion.Message) + .Build(); + + return new OllamaResponse(prompt); + } + + private async Task StartProcess(Uri baseAddress) + { + var processStartInfo = new ProcessStartInfo(await _ollamaExePath); + + processStartInfo.Arguments += $"serve "; + processStartInfo.EnvironmentVariables.Add("OLLAMA_HOST", baseAddress.ToString()); + + processStartInfo.UseShellExecute = false; + processStartInfo.CreateNoWindow = true; + processStartInfo.RedirectStandardError = _cellmConfiguration.Debug; + processStartInfo.RedirectStandardOutput = _cellmConfiguration.Debug; + + var process = Process.Start(processStartInfo) ?? throw new CellmException("Failed to run Ollama"); + + if (_cellmConfiguration.Debug) + { + process.OutputDataReceived += (sender, e) => + { + if (!string.IsNullOrEmpty(e.Data)) + { + _logger.LogDebug(e.Data); + Debug.WriteLine(e.Data); + } + }; + + process.BeginOutputReadLine(); + process.BeginErrorReadLine(); + } + + var address = new Uri(baseAddress, "/v1/models"); + await _localUtilities.WaitForServer(address, process); + + // Kill Ollama when Excel exits or dies + _processManager.AssignProcessToExcel(process); + + return process; + } + + private async Task GetChatClient(Uri address, string modelId) + { + // Download model if it doesn't exist + var models = await _httpClient.GetFromJsonAsync>("api/tags") ?? throw new CellmException(); + + if (!models.Select(x => x.Name).Contains(modelId)) + { + var body = new StringContent($"{{\"model\":\"{modelId}\", \"stream\": \"false\"}}", Encoding.UTF8, "application/json"); + var response = await _httpClient.PostAsync("api/pull", body); + response.EnsureSuccessStatusCode(); + } + + return new ChatClientBuilder() + .UseLogging() + .UseFunctionInvocation() + .Use(new OllamaChatClient(address, modelId, _httpClient)); + } +} diff --git a/src/Cellm/Models/Ollama/OllamaResponse.cs b/src/Cellm/Models/Ollama/OllamaResponse.cs new file mode 100644 index 0000000..475d7f9 --- /dev/null +++ b/src/Cellm/Models/Ollama/OllamaResponse.cs @@ -0,0 +1,5 @@ +using Cellm.Prompts; + +namespace Cellm.Models.Ollama; + +internal record OllamaResponse(Prompt Prompt) : IModelResponse; \ No newline at end of file diff --git a/src/Cellm/Models/Providers.cs b/src/Cellm/Models/Providers.cs index e0e30fa..9f30d8d 100644 --- a/src/Cellm/Models/Providers.cs +++ b/src/Cellm/Models/Providers.cs @@ -4,5 +4,6 @@ public enum Providers { Anthropic, Llamafile, + Ollama, OpenAi } diff --git a/src/Cellm/Services/ServiceLocator.cs b/src/Cellm/Services/ServiceLocator.cs index 545f5a6..9ac7180 100644 --- a/src/Cellm/Services/ServiceLocator.cs +++ b/src/Cellm/Services/ServiceLocator.cs @@ -4,10 +4,11 @@ using Cellm.Models; using Cellm.Models.Anthropic; using Cellm.Models.Llamafile; +using Cellm.Models.Local; using Cellm.Models.ModelRequestBehavior; +using Cellm.Models.Ollama; using Cellm.Models.OpenAi; using Cellm.Services.Configuration; -using Cellm.Tools; using Cellm.Tools.FileReader; using ExcelDna.Integration; using MediatR; @@ -46,6 +47,7 @@ private static IServiceCollection ConfigureServices(IServiceCollection services) services .Configure(configuration.GetRequiredSection(nameof(CellmConfiguration))) .Configure(configuration.GetRequiredSection(nameof(AnthropicConfiguration))) + .Configure(configuration.GetRequiredSection(nameof(OllamaConfiguration))) .Configure(configuration.GetRequiredSection(nameof(OpenAiConfiguration))) .Configure(configuration.GetRequiredSection(nameof(LlamafileConfiguration))) .Configure(configuration.GetRequiredSection(nameof(RateLimiterConfiguration))) @@ -77,7 +79,6 @@ private static IServiceCollection ConfigureServices(IServiceCollection services) sentryLoggingOptions.Environment = sentryConfiguration.Environment; sentryLoggingOptions.AutoSessionTracking = true; sentryLoggingOptions.IsGlobalModeEnabled = true; - sentryLoggingOptions.ExperimentalMetrics = new ExperimentalMetricsOptions { EnableCodeLocations = true }; sentryLoggingOptions.AddIntegration(new ProfilingIntegration()); }); }); @@ -85,11 +86,17 @@ private static IServiceCollection ConfigureServices(IServiceCollection services) // Internals services .AddSingleton(configuration) - .AddMemoryCache() .AddMediatR(cfg => cfg.RegisterServicesFromAssembly(Assembly.GetExecutingAssembly())) .AddTransient() .AddSingleton() - .AddSingleton(); + .AddSingleton() + .AddSingleton() + .AddSingleton(); ; + +#pragma warning disable EXTEXP0018 // Type is for evaluation purposes only and is subject to change or removal in future updates. + services + .AddHybridCache(); +#pragma warning restore EXTEXP0018 // Type is for evaluation purposes only and is subject to change or removal in future updates. // Tools services @@ -121,6 +128,15 @@ private static IServiceCollection ConfigureServices(IServiceCollection services) anthropicHttpClient.Timeout = TimeSpan.FromHours(1); }).AddResilienceHandler($"{nameof(AnthropicRequestHandler)}ResiliencePipeline", resiliencePipelineConfigurator.ConfigureResiliencePipeline); + var ollamaConfiguration = configuration.GetRequiredSection(nameof(OllamaConfiguration)).Get() + ?? throw new NullReferenceException(nameof(OllamaConfiguration)); + + services.AddHttpClient, OllamaRequestHandler>(ollamaHttpClient => + { + ollamaHttpClient.BaseAddress = ollamaConfiguration.BaseAddress; + ollamaHttpClient.Timeout = TimeSpan.FromHours(1); + }).AddResilienceHandler($"{nameof(OllamaRequestHandler)}ResiliencePipeline", resiliencePipelineConfigurator.ConfigureResiliencePipeline); + var openAiConfiguration = configuration.GetRequiredSection(nameof(OpenAiConfiguration)).Get() ?? throw new NullReferenceException(nameof(OpenAiConfiguration)); @@ -131,10 +147,6 @@ private static IServiceCollection ConfigureServices(IServiceCollection services) openAiHttpClient.Timeout = TimeSpan.FromHours(1); }).AddResilienceHandler($"{nameof(OpenAiRequestHandler)}ResiliencePipeline", resiliencePipelineConfigurator.ConfigureResiliencePipeline); - services - .AddSingleton() - .AddSingleton(); - // Model request pipeline services .AddSingleton(typeof(IPipelineBehavior<,>), typeof(SentryBehavior<,>)) diff --git a/src/Cellm/appsettings.Local.Ollama.json b/src/Cellm/appsettings.Local.Ollama.json index ea8d8b5..d6171e2 100644 --- a/src/Cellm/appsettings.Local.Ollama.json +++ b/src/Cellm/appsettings.Local.Ollama.json @@ -1,8 +1,5 @@ { - "OpenAiConfiguration": { - "BaseAddress": "http://localhost:11434" - }, "CellmConfiguration": { - "DefaultProvider": "OpenAI" + "DefaultProvider": "Ollama" } } diff --git a/src/Cellm/appsettings.json b/src/Cellm/appsettings.json index ad34270..42cb4c1 100644 --- a/src/Cellm/appsettings.json +++ b/src/Cellm/appsettings.json @@ -4,9 +4,10 @@ "DefaultModel": "claude-3-5-sonnet-latest", "Version": "2023-06-01" }, - "GoogleAiConfiguration": { - "BaseAddress": "https://generativelanguage.googleapis.com", - "DefaultModel": "gemini-1.5-flash-latest" + "OllamaConfiguration": { + "OllamaUri": "https://github.com/ollama/ollama/releases/download/v0.4.2/ollama-windows-amd64.zip", + "BaseAddress": "http://localhost:11434", + "DefaultModel": "gemma-2:2b" }, "OpenAiConfiguration": { "BaseAddress": "https://api.openai.com",