From 3d136428694343fc62fcdedc08849adc622acd6e Mon Sep 17 00:00:00 2001 From: Kasper Marstal Date: Wed, 22 Jan 2025 19:57:06 +0100 Subject: [PATCH] docs: Fix appsettings.Local.*.json examples and update readme to match --- README.md | 111 +++++++++++------- .../Anthropic/AnthropicConfiguration.cs | 2 + .../Providers/Mistral/MistralConfiguration.cs | 10 ++ .../Providers/Ollama/OllamaConfiguration.cs | 2 +- .../Providers/OpenAi/OpenAiConfiguration.cs | 2 + .../Providers/OpenAi/OpenAiRequestHandler.cs | 3 +- .../OpenAiCompatibleConfiguration.cs | 2 + .../OpenAiCompatibleRequestHandler.cs | 4 +- .../Models/ServiceCollectionExtensions.cs | 4 +- src/Cellm/appsettings.Local.Anthropic.json | 6 +- src/Cellm/appsettings.Local.Google.json | 5 +- src/Cellm/appsettings.Local.Llamafile.json | 3 +- src/Cellm/appsettings.Local.Mistral.json | 7 +- src/Cellm/appsettings.Local.Ollama.json | 2 +- src/Cellm/appsettings.Local.OpenAi.json | 5 +- src/Cellm/appsettings.Local.vLLM.json | 2 +- src/Cellm/appsettings.json | 3 +- 17 files changed, 103 insertions(+), 70 deletions(-) create mode 100644 src/Cellm/Models/Providers/Mistral/MistralConfiguration.cs diff --git a/README.md b/README.md index 6bfb8e6..abc2f04 100644 --- a/README.md +++ b/README.md @@ -11,7 +11,7 @@ Cellm is an Excel extension that lets you use Large Language Models (LLMs) like ## What is Cellm? Similar to Excel's `=SUM()` function that outputs the sum of a range of numbers, Cellm's `=PROMPT()` function outputs the AI response to a range of text. -For example, you can write `=PROMPT(A1:A10, "Extract all person names mentioned in the text.")` in a cell's formula and drag the cell to apply the prompt to many rows. Cellm is useful when you want to use AI for repetitive tasks that would normally require copy-pasting data in and out of a chat window many times. +For example, you can write `=PROMPT(A1, "Extract all person names mentioned in the text.")` in a cell's formula and drag the cell to apply the prompt to many rows. Cellm is useful when you want to use AI for repetitive tasks that would normally require copy-pasting data in and out of a chat window many times. ## Key features This extension does one thing and one thing well. @@ -30,7 +30,7 @@ In this example, we copy the papers' titles and abstracts into Excel and write t We then use autofill to apply the prompt to many papers. Simple and powerful. -A single paper is misclassified because the original inclusion and exclusion criteria were summarized in one sentence. This is a good example, however, because it shows that these models rely entirely on your input and can make mistakes. +Note that a paper is misclassified. The models _will_ make mistakes. It is your responsibility to cross validate if a model is accurate enough for your use case and upgrade model or use another approach if not. ## Getting Started @@ -39,7 +39,7 @@ Cellm must be built from source and installed via Excel. Follow the steps below. ### Requirements - Windows -- [.NET 6.0 SDK](https://dotnet.microsoft.com/en-us/download/dotnet/6.0) +- [.NET 9.0 SDK](https://dotnet.microsoft.com/en-us/download/dotnet/9.0) - [Excel 2010 or higher (desktop app)](https://www.microsoft.com/en-us/microsoft-365/excel) ### Build @@ -54,32 +54,30 @@ Cellm must be built from source and installed via Excel. Follow the steps below. cd cellm ``` -3. Add your Anthropic API key. Rename `src/Cellm/appsettings.Anthropic.json` to `src/Cellm/appsettings.Local.json` and insert it. Example: - ```json - { - "AnthropicConfiguration": { - "ApiKey": "YOUR_ANTHROPIC_APIKEY" - } - } - ``` - - Cellm uses Anthropic as the default model provider. You can also use models from OpenAI, Mistral, Google, or run models locally. See the `appsettings.Local.*.json` files for examples. - -4. Install dependencies: +3. Install dependencies: ```cmd dotnet restore ``` -5. Build the project: +4. Build the project: ```cmd dotnet build --configuration Release ``` +5. Cellm uses Ollama and the Gemma 2 2B model by default. Download and install [Ollama](https://ollama.com/) and run the following command in your Windows terminal to download the model: + ```cmd + ollama pull gemma2:2b + ``` + + To use other models, see the [Models](#models) section below. + +These steps will build Cellm on your computer. Continue with the steps below to install Cellm in Excel. + ### Install 1. In Excel, go to File > Options > Add-Ins. 2. In the `Manage` drop-down menu, select `Excel Add-ins` and click `Go...`. -3. Click `Browse...` and select the `Cellm-AddIn64.xll` file in the bin/Release/net6.0-windows folder. +3. Click `Browse...` and select the `Cellm-AddIn64.xll` file in the bin/Release/net9.0-windows. This folder is located in the root of git repository that you cloned. 4. Check the box next to Cellm and click `OK`. ## Usage @@ -166,60 +164,84 @@ Cellm is useful for repetitive tasks on both structured and unstructured data. H These use cases are starting points. Experiment with different instructions to find what works best for your data. It works best when combined with human judgment and expertise in your specific domain. -## Run Models Locally +## Models -### Requirements +### Hosted LLMs + +Cellm supports hosted models from Anthropic, Google, OpenAI, and any OpenAI-compatible provider. To use e.g. Claude 3.5 Sonnet from Anthropic: -- [Docker](https://www.docker.com/products/docker-desktop/) (optional) -- A GPU and [NVIDIA CUDA Toolkit 12.4](https://developer.nvidia.com/cuda-downloads) or higher (optional) +1. Rename `src/Cellm/appsettings.Anthropic.json` to `src/Cellm/appsettings.Local.json`. + +2. Add your Anthropic API key to `src/Cellm/appsettings.Local.json`: + ```json + { + "AnthropicConfiguration": { + "ApiKey": "ADD_YOUR_ANTHROPIC_API_KEY_HERE" + }, + "ProviderConfiguration": { + "DefaultProvider": "Anthropic" + } + } + ``` + +See the `appsettings.Local.*.json` files for examples on other providers. + +### Local LLMs -#### Local LLMs +Cellm supports local models that run on your computer via Llamafiles, Ollama, or vLLM. This ensures none of your data ever leaves your machine. And it's free. -Cellm can run LLM models locally on your computer via Llamafiles, Ollama, or vLLM. This ensures none of your data ever leaves your machine. And it's free. +#### Ollama -Cellm uses Gemma 2 2B model with 4-bit quantization by default. This clever little model runs fine on a CPU. +Cellm uses Ollama Gemma 2 2B model by default. This clever little model runs fine on a CPU. For any model larger than 3B you will need a GPU. Ollama will automatically use your GPU if you have one. To get started: -For Ollama and vLLM you will need docker, and for models larger than 3B you will need a GPU. +1. Download and install Ollama from [https://ollama.com/](https://ollama.com/). +2. Download the model by running the following command in your Windows terminal: + ```cmd + ollama pull gemma2:2b + ``` + +See [https://ollama.com/search](https://ollama.com/search) for a complete list of supported models. -### LLamafile +#### LLamafile -Llamafile is a stand-alone executable that is very easy to setup. Cellm will automatically download a Llamafile model and start a Llamafile server the first time you call `=PROMPT()`. +Llamafile is a stand-alone executable that is very easy to setup. To get started: -To get started: +1. Download a llamafile from https://github.com/Mozilla-Ocho/llamafile (e.g. [Gemma 2 2B](https://huggingface.co/Mozilla/gemma-2-2b-it-llamafile/blob/main/gemma-2-2b-it.Q6_K.llamafile?download=true)). +2. Run the following command in your Windows terminal: + ```cmd + .\gemma-2-2b-it.Q6_K.llamafile.exe --server --v2 + ``` -1. Rename `appsettings.Llamafile.json` to `appsettings.Local.json`. -2. Build and install Cellm. -3. Run e.g. `=PROMPT(A1, "Extract keywords")` in a formula. -4. Wait 5-10 min depending on your internet connection. The model will reply once it is ready. + To offload inference to your NVIDIA or AMD GPU, run: -This will use the Llama 3.2 1B model. To use other models, edit the appsettings file and rebuild. + ```cmd + .\gemma-2-2b-it.Q6_K.llamafile.exe --server --v2 -ngl 999 + ``` -Use `appsettings.Llamafile.GPU.json` to offload Llamafile inference to your NVIDIA or AMD GPU. +3. Rename `appsettings.Llamafile.json` to `appsettings.Local.json`. +4. Build and install Cellm. -### Ollama and vLLM +### Dockerized Ollama and vLLM -Ollama and vLLM are LLM inference servers for running models locally. Ollama is designed for easy of use and vLLM is designed to run models efficiently with high throughput. Both Ollama and vLLM are packaged up with docker compose files in the `docker/` folder. +Both Ollama and vLLM are packaged up with docker compose files in the `docker/` folder. Ollama is designed for easy of use and vLLM is designed to run many requests in parallel which is particularly useful for this Add-In. Open WebUI in included in both Ollama and vLLM docker compose files so you can test the local model outside of Cellm. It is available at `http://localhost:3000`. To get started, we recommend using Ollama with the Gemma 2 2B model: -1. Rename `appsettings.Ollama.json` to `appsettings.Local.json`, -2. Build and install Cellm. -3. Run the following command in the `docker/` directory: +1. Build and install Cellm. +2. Run the following command in the `docker/` directory: ```cmd docker compose -f docker-compose.Ollama.yml up --detach docker compose -f docker-compose.Ollama.yml exec backend ollama pull gemma2:2b docker compose -f docker-compose.Ollama.yml down // When you want to shut it down ``` -If you want to speed up inference, you can use your GPU as well: +To use other Ollama models, pull another of the [supported models](https://ollama.com/search). If you want to speed up inference, you can use your GPU as well: ```cmd docker compose -f docker-compose.Ollama.yml -f docker-compose.Ollama.GPU.yml up --detach ``` -A GPU is practically required if you want to use models larger than Gemma 2 2b. - -If you want to speed up running many requests in parallel, you can use vLLM instead of Ollama. You must supply the docker compose file with a Huggingface API key either via an environment variable or editing the docker compose file directy. Look at the vLLM docker compose file for details. If you don't know what a Huggingface API key is, just use Ollama. +If you want to further speed up running many requests in parallel, you can use vLLM instead of Ollama. You must supply the docker compose file with a Huggingface API key either via an environment variable or editing the docker compose file directy. Look at the vLLM docker compose file for details. If you don't know what a Huggingface API key is, just use Ollama. To start vLLM: @@ -227,9 +249,8 @@ To start vLLM: docker compose -f docker-compose.vLLM.GPU.yml up --detach ``` -To use other Ollama models, pull another of the [supported models](https://ollama.com/library). To use other vLLM models, change the "--model" argument to another Huggingface model. +To use other vLLM models, change the "--model" argument in the docker compose file to another Huggingface model. -Open WebUI in included in both Ollama and vLLM docker compose files so you can test the local model outside of Cellm. It is available at `http://localhost:3000`. ## Dos and Don'ts Do: diff --git a/src/Cellm/Models/Providers/Anthropic/AnthropicConfiguration.cs b/src/Cellm/Models/Providers/Anthropic/AnthropicConfiguration.cs index 598b6b9..b2580e1 100644 --- a/src/Cellm/Models/Providers/Anthropic/AnthropicConfiguration.cs +++ b/src/Cellm/Models/Providers/Anthropic/AnthropicConfiguration.cs @@ -10,6 +10,8 @@ internal class AnthropicConfiguration : IProviderConfiguration public string ApiKey { get; init; } + public List Models { get; init; } = new(); + public AnthropicConfiguration() { BaseAddress = default!; diff --git a/src/Cellm/Models/Providers/Mistral/MistralConfiguration.cs b/src/Cellm/Models/Providers/Mistral/MistralConfiguration.cs new file mode 100644 index 0000000..012e527 --- /dev/null +++ b/src/Cellm/Models/Providers/Mistral/MistralConfiguration.cs @@ -0,0 +1,10 @@ +namespace Cellm.Models.Providers.Mistral; + +internal class MistralConfiguration : IProviderConfiguration +{ + public Uri BaseAddress { get; init; } = default!; + + public string DefaultModel { get; init; } = string.Empty; + + public List Models { get; init; } = new(); +} \ No newline at end of file diff --git a/src/Cellm/Models/Providers/Ollama/OllamaConfiguration.cs b/src/Cellm/Models/Providers/Ollama/OllamaConfiguration.cs index ac9e72a..1612be0 100644 --- a/src/Cellm/Models/Providers/Ollama/OllamaConfiguration.cs +++ b/src/Cellm/Models/Providers/Ollama/OllamaConfiguration.cs @@ -8,5 +8,5 @@ internal class OllamaConfiguration : IProviderConfiguration public string DefaultModel { get; init; } = string.Empty; - public bool EnableServer { get; init; } = false; + public List Models { get; init; } = new(); } \ No newline at end of file diff --git a/src/Cellm/Models/Providers/OpenAi/OpenAiConfiguration.cs b/src/Cellm/Models/Providers/OpenAi/OpenAiConfiguration.cs index 3bd69d8..511ad86 100644 --- a/src/Cellm/Models/Providers/OpenAi/OpenAiConfiguration.cs +++ b/src/Cellm/Models/Providers/OpenAi/OpenAiConfiguration.cs @@ -5,4 +5,6 @@ internal class OpenAiConfiguration : IProviderConfiguration public string DefaultModel { get; init; } = string.Empty; public string ApiKey { get; init; } = string.Empty; + + public List Models { get; init; } = new(); } diff --git a/src/Cellm/Models/Providers/OpenAi/OpenAiRequestHandler.cs b/src/Cellm/Models/Providers/OpenAi/OpenAiRequestHandler.cs index 519a5ac..f6ad293 100644 --- a/src/Cellm/Models/Providers/OpenAi/OpenAiRequestHandler.cs +++ b/src/Cellm/Models/Providers/OpenAi/OpenAiRequestHandler.cs @@ -4,7 +4,8 @@ namespace Cellm.Models.Providers.OpenAi; -internal class OpenAiRequestHandler([FromKeyedServices(Provider.OpenAi)] IChatClient chatClient) : IModelRequestHandler +internal class OpenAiRequestHandler( + [FromKeyedServices(Provider.OpenAi)] IChatClient chatClient) : IModelRequestHandler { public async Task Handle(OpenAiRequest request, CancellationToken cancellationToken) diff --git a/src/Cellm/Models/Providers/OpenAiCompatible/OpenAiCompatibleConfiguration.cs b/src/Cellm/Models/Providers/OpenAiCompatible/OpenAiCompatibleConfiguration.cs index 2c7a83c..e75fc97 100644 --- a/src/Cellm/Models/Providers/OpenAiCompatible/OpenAiCompatibleConfiguration.cs +++ b/src/Cellm/Models/Providers/OpenAiCompatible/OpenAiCompatibleConfiguration.cs @@ -7,4 +7,6 @@ internal class OpenAiCompatibleConfiguration public string DefaultModel { get; init; } = string.Empty; public string ApiKey { get; init; } = string.Empty; + + public int HttpTimeoutInSeconds = 3600; } diff --git a/src/Cellm/Models/Providers/OpenAiCompatible/OpenAiCompatibleRequestHandler.cs b/src/Cellm/Models/Providers/OpenAiCompatible/OpenAiCompatibleRequestHandler.cs index a08e974..3a51b36 100644 --- a/src/Cellm/Models/Providers/OpenAiCompatible/OpenAiCompatibleRequestHandler.cs +++ b/src/Cellm/Models/Providers/OpenAiCompatible/OpenAiCompatibleRequestHandler.cs @@ -1,11 +1,9 @@ using Cellm.Models.Prompts; -using Microsoft.Extensions.Options; namespace Cellm.Models.Providers.OpenAiCompatible; internal class OpenAiCompatibleRequestHandler( - OpenAiCompatibleChatClientFactory openAiCompatibleChatClientFactory, - IOptions openAiCompatibleConfiguration) + OpenAiCompatibleChatClientFactory openAiCompatibleChatClientFactory) : IModelRequestHandler { diff --git a/src/Cellm/Models/ServiceCollectionExtensions.cs b/src/Cellm/Models/ServiceCollectionExtensions.cs index 36a1240..73c8748 100644 --- a/src/Cellm/Models/ServiceCollectionExtensions.cs +++ b/src/Cellm/Models/ServiceCollectionExtensions.cs @@ -74,7 +74,9 @@ public static IServiceCollection AddOpenAiCompatibleChatClient(this IServiceColl .AddSingleton() .AddHttpClient(openAiCompatibleHttpClient => { - openAiCompatibleHttpClient.Timeout = Timeout.InfiniteTimeSpan; + openAiCompatibleHttpClient.Timeout = TimeSpan.FromSeconds(configuration + .GetSection(nameof(ProviderConfiguration)) + .GetValue(nameof(ProviderConfiguration.HttpTimeoutInSeconds))); }) .AddResilienceHandler(nameof(OpenAiCompatibleChatClientFactory), resiliencePipelineConfigurator.ConfigureResiliencePipeline); diff --git a/src/Cellm/appsettings.Local.Anthropic.json b/src/Cellm/appsettings.Local.Anthropic.json index e8f2ea5..57a75ed 100644 --- a/src/Cellm/appsettings.Local.Anthropic.json +++ b/src/Cellm/appsettings.Local.Anthropic.json @@ -1,10 +1,8 @@ { "AnthropicConfiguration": { - "DefaultModel": "claude-3-5-sonnet-20241022", "ApiKey": "YOUR_ANTHROPIC_API_KEY" }, - "CellmConfiguration": { - "DefaultProvider": "Anthropic", - "HttpTimeoutInSeconds": 30 + "ProviderConfiguration": { + "DefaultProvider": "Anthropic" } } diff --git a/src/Cellm/appsettings.Local.Google.json b/src/Cellm/appsettings.Local.Google.json index 4e84c5b..d027d38 100644 --- a/src/Cellm/appsettings.Local.Google.json +++ b/src/Cellm/appsettings.Local.Google.json @@ -5,8 +5,7 @@ "ApiKey": "YOUR_GEMINI_API_KEY" }, - "CellmConfiguration": { - "DefaultProvider": "OpenAiCompatible", - "HttpTimeoutInSeconds": 30 + "ProviderConfiguration": { + "DefaultProvider": "OpenAiCompatible" } } diff --git a/src/Cellm/appsettings.Local.Llamafile.json b/src/Cellm/appsettings.Local.Llamafile.json index e3de534..f8779c7 100644 --- a/src/Cellm/appsettings.Local.Llamafile.json +++ b/src/Cellm/appsettings.Local.Llamafile.json @@ -1,5 +1,6 @@ { - "CellmConfiguration": { + "ProviderConfiguration": { + "BaseAddress": "http://127.0.0.1:8080", "DefaultProvider": "Llamafile" } } diff --git a/src/Cellm/appsettings.Local.Mistral.json b/src/Cellm/appsettings.Local.Mistral.json index d982055..3bd8972 100644 --- a/src/Cellm/appsettings.Local.Mistral.json +++ b/src/Cellm/appsettings.Local.Mistral.json @@ -1,11 +1,10 @@ { - "OpenAiCompatibleConfiguration": { + "MistralConfiguration": { "BaseAddress": "https://api.mistral.ai/v1", "DefaultModel": "mistral-small-latest", "ApiKey": "YOUR_MISTRAL_API_KEY" }, - "CellmConfiguration": { - "DefaultProvider": "OpenAiCompatible", - "HttpTimeoutInSeconds": 30 + "ProviderConfiguration": { + "DefaultProvider": "OpenAiCompatible" } } diff --git a/src/Cellm/appsettings.Local.Ollama.json b/src/Cellm/appsettings.Local.Ollama.json index d6171e2..9febe4b 100644 --- a/src/Cellm/appsettings.Local.Ollama.json +++ b/src/Cellm/appsettings.Local.Ollama.json @@ -1,5 +1,5 @@ { - "CellmConfiguration": { + "ProviderConfiguration": { "DefaultProvider": "Ollama" } } diff --git a/src/Cellm/appsettings.Local.OpenAi.json b/src/Cellm/appsettings.Local.OpenAi.json index de345fc..fbcda8e 100644 --- a/src/Cellm/appsettings.Local.OpenAi.json +++ b/src/Cellm/appsettings.Local.OpenAi.json @@ -3,8 +3,7 @@ "DefaultModel": "gpt-4o-mini", "ApiKey": "YOUR_OPENAI_API_KEY" }, - "CellmConfiguration": { - "DefaultProvider": "OpenAI", - "HttpTimeoutInSeconds": 30 + "ProviderConfiguration": { + "DefaultProvider": "OpenAI" } } diff --git a/src/Cellm/appsettings.Local.vLLM.json b/src/Cellm/appsettings.Local.vLLM.json index ba3d74a..c834681 100644 --- a/src/Cellm/appsettings.Local.vLLM.json +++ b/src/Cellm/appsettings.Local.vLLM.json @@ -2,7 +2,7 @@ "OpenAiCompatibleConfiguration": { "BaseAddress": "http://localhost:8000/v1" }, - "CellmConfiguration": { + "ProviderConfiguration": { "DefaultProvider": "OpenAiCompatible" } } diff --git a/src/Cellm/appsettings.json b/src/Cellm/appsettings.json index 58824a2..58b0e98 100644 --- a/src/Cellm/appsettings.json +++ b/src/Cellm/appsettings.json @@ -9,7 +9,6 @@ "DefaultModel": "gemma-2-2b" }, "OllamaConfiguration": { - "ZipUrl": "https://github.com/ollama/ollama/releases/download/v0.4.2/ollama-windows-amd64.zip", "BaseAddress": "http://127.0.0.1:11434", "DefaultModel": "gemma2:2b" }, @@ -25,7 +24,7 @@ "DefaultModel": "gemma2:2b", "DefaultTemperature": 0, "CacheTimeoutInSeconds": 3600, - "HttpTimeoutInSeconds": 600, + "HttpTimeoutInSeconds": 3600, "EnableCache": true, "EnableTools": false },