diff --git a/src/Cellm/Cellm.csproj b/src/Cellm/Cellm.csproj index a2abfb7..0164082 100644 --- a/src/Cellm/Cellm.csproj +++ b/src/Cellm/Cellm.csproj @@ -26,7 +26,6 @@ - diff --git a/src/Cellm/Models/ModelRequestBehavior/CachingBehavior.cs b/src/Cellm/Models/ModelRequestBehavior/CachingBehavior.cs index e517d70..5712f77 100644 --- a/src/Cellm/Models/ModelRequestBehavior/CachingBehavior.cs +++ b/src/Cellm/Models/ModelRequestBehavior/CachingBehavior.cs @@ -1,51 +1,32 @@ -using System.Security.Cryptography; -using System.Text; -using System.Text.Json; +using System.Text.Json; using Cellm.Services.Configuration; using MediatR; -using Microsoft.Extensions.Caching.Memory; +using Microsoft.Extensions.Caching.Hybrid; using Microsoft.Extensions.Options; namespace Cellm.Models.ModelRequestBehavior; -internal class CachingBehavior : IPipelineBehavior +internal class CachingBehavior(HybridCache cache, IOptions cellmConfiguration) : IPipelineBehavior where TRequest : IModelRequest where TResponse : IModelResponse { - private readonly IMemoryCache _memoryCache; - private readonly MemoryCacheEntryOptions _memoryCacheEntryOptions; - - public CachingBehavior(IMemoryCache memoryCache, IOptions _cellmConfiguration) + private readonly HybridCacheEntryOptions _cacheEntryOptions = new() { - _memoryCache = memoryCache; - _memoryCacheEntryOptions = new() - { - SlidingExpiration = TimeSpan.FromSeconds(_cellmConfiguration.Value.CacheTimeoutInSeconds) - }; - } + Expiration = TimeSpan.FromSeconds(cellmConfiguration.Value.CacheTimeoutInSeconds) + }; public async Task Handle(TRequest request, RequestHandlerDelegate next, CancellationToken cancellationToken) { - var key = GetKey(request); - - if (_memoryCache.TryGetValue(key, out object? value) && value is TResponse response) + if (cellmConfiguration.Value.EnableCache) { - return response; + return await cache.GetOrCreateAsync( + JsonSerializer.Serialize(request.Prompt), + async cancel => await next(), + options: _cacheEntryOptions, + cancellationToken: cancellationToken + ); } - response = await next(); - - _memoryCache.Set(key, response, _memoryCacheEntryOptions); - - return response; - } - - private static string GetKey(T key) - { - var json = JsonSerializer.Serialize(key); - var bytes = SHA256.HashData(Encoding.UTF8.GetBytes(json)); - var hash = Convert.ToHexString(bytes); - - return hash; + return await next(); } } diff --git a/src/Cellm/Services/Configuration/CellmConfiguration.cs b/src/Cellm/Services/Configuration/CellmConfiguration.cs index 642530b..8253ce1 100644 --- a/src/Cellm/Services/Configuration/CellmConfiguration.cs +++ b/src/Cellm/Services/Configuration/CellmConfiguration.cs @@ -14,6 +14,8 @@ public class CellmConfiguration public int CacheTimeoutInSeconds { get; init; } + public bool EnableCache { get; init; } + public bool EnableTools { get; init; } } diff --git a/src/Cellm/appsettings.json b/src/Cellm/appsettings.json index c0d94f9..98fe061 100644 --- a/src/Cellm/appsettings.json +++ b/src/Cellm/appsettings.json @@ -31,6 +31,7 @@ "DefaultTemperature": 0, "HttpTimeoutInSeconds": 600, "CacheTimeoutInSeconds": 3600, + "EnableCache": true, "EnableTools": true }, "RateLimiterConfiguration": { diff --git a/src/Cellm/packages.lock.json b/src/Cellm/packages.lock.json index 869c651..1ae3b88 100644 --- a/src/Cellm/packages.lock.json +++ b/src/Cellm/packages.lock.json @@ -83,19 +83,6 @@ "Microsoft.Extensions.Options": "9.0.0" } }, - "Microsoft.Extensions.Caching.Memory": { - "type": "Direct", - "requested": "[9.0.0, )", - "resolved": "9.0.0", - "contentHash": "zbnPX/JQ0pETRSUG9fNPBvpIq42Aufvs15gGYyNIMhCun9yhmWihz0WgsI7bSDPjxWTKBf8oX/zv6v2uZ3W9OQ==", - "dependencies": { - "Microsoft.Extensions.Caching.Abstractions": "9.0.0", - "Microsoft.Extensions.DependencyInjection.Abstractions": "9.0.0", - "Microsoft.Extensions.Logging.Abstractions": "9.0.0", - "Microsoft.Extensions.Options": "9.0.0", - "Microsoft.Extensions.Primitives": "9.0.0" - } - }, "Microsoft.Extensions.Configuration": { "type": "Direct", "requested": "[9.0.0, )", @@ -298,6 +285,18 @@ "Microsoft.Extensions.Primitives": "9.0.0" } }, + "Microsoft.Extensions.Caching.Memory": { + "type": "Transitive", + "resolved": "9.0.0", + "contentHash": "zbnPX/JQ0pETRSUG9fNPBvpIq42Aufvs15gGYyNIMhCun9yhmWihz0WgsI7bSDPjxWTKBf8oX/zv6v2uZ3W9OQ==", + "dependencies": { + "Microsoft.Extensions.Caching.Abstractions": "9.0.0", + "Microsoft.Extensions.DependencyInjection.Abstractions": "9.0.0", + "Microsoft.Extensions.Logging.Abstractions": "9.0.0", + "Microsoft.Extensions.Options": "9.0.0", + "Microsoft.Extensions.Primitives": "9.0.0" + } + }, "Microsoft.Extensions.Compliance.Abstractions": { "type": "Transitive", "resolved": "9.0.0",