Skip to content

Commit

Permalink
Merge branch 'SciSharp:master' into master
Browse files Browse the repository at this point in the history
  • Loading branch information
@Oceania2018
Oceania2018 authored Oct 2, 2024
2 parents 3439809 + 7b649c0 commit 022c050
Show file tree
Hide file tree
Showing 18 changed files with 157 additions and 123 deletions.
4 changes: 2 additions & 2 deletions src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeHook.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,9 +5,9 @@ public interface IKnowledgeHook
Task<List<KnowledgeChunk>> CollectChunkedKnowledge()
=> Task.FromResult(new List<KnowledgeChunk>());

Task<List<string>> GetRelevantKnowledges(string text)
Task<List<string>> GetRelevantKnowledges(RoleDialogModel message, string text)
=> Task.FromResult(new List<string>());

Task<List<string>> GetGlobalKnowledges()
Task<List<string>> GetGlobalKnowledges(RoleDialogModel message)
=> Task.FromResult(new List<string>());
}
21 changes: 14 additions & 7 deletions src/Plugins/BotSharp.Plugin.AnthropicAI/Providers/ChatCompletionProvider.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -50,10 +50,10 @@ public async Task<RoleDialogModel> GetChatCompletions(Agent agent, List<RoleDial
{
var toolResult = response.Content.OfType<ToolUseContent>().First();

responseMessage = new RoleDialogModel(AgentRole.Function, response.FirstMessage?.Text)
responseMessage = new RoleDialogModel(AgentRole.Function, response.FirstMessage?.Text ?? string.Empty)
{
CurrentAgentId = agent.Id,
MessageId = conversations.Last().MessageId,
MessageId = conversations.LastOrDefault()?.MessageId ?? string.Empty,
ToolCallId = toolResult.Id,
FunctionName = toolResult.Name,
FunctionArgs = JsonSerializer.Serialize(toolResult.Input)
Expand All @@ -62,10 +62,10 @@ public async Task<RoleDialogModel> GetChatCompletions(Agent agent, List<RoleDial
else
{
var message = response.FirstMessage;
responseMessage = new RoleDialogModel(AgentRole.Assistant, message.Text)
responseMessage = new RoleDialogModel(AgentRole.Assistant, message?.Text ?? string.Empty)
{
CurrentAgentId = agent.Id,
MessageId = conversations.Last().MessageId
MessageId = conversations.LastOrDefault()?.MessageId ?? string.Empty,
};
}

Expand All @@ -77,8 +77,8 @@ public async Task<RoleDialogModel> GetChatCompletions(Agent agent, List<RoleDial
Prompt = prompt,
Provider = Provider,
Model = _model,
PromptCount = response.Usage.InputTokens,
CompletionCount = response.Usage.OutputTokens
PromptCount = response.Usage?.InputTokens ?? 0,
CompletionCount = response.Usage?.OutputTokens ?? 0
});
}

Expand Down Expand Up @@ -120,7 +120,14 @@ public Task<bool> GetChatCompletionsStreamingAsync(Agent agent, List<RoleDialogM
prompt += "\r\n\r\n" + response_with_function;*/

var messages = new List<Message>();
foreach (var conv in conversations)
var filteredMessages = conversations.Select(x => x).ToList();
var firstUserMsgIdx = filteredMessages.FindIndex(x => x.Role == AgentRole.User);
if (firstUserMsgIdx > 0)
{
filteredMessages = filteredMessages.Where((_, idx) => idx >= firstUserMsgIdx).ToList();
}

foreach (var conv in filteredMessages)
{
if (conv.Role == AgentRole.User)
{
Expand Down
2 changes: 1 addition & 1 deletion src/Plugins/BotSharp.Plugin.AzureOpenAI/BotSharp.Plugin.AzureOpenAI.csproj
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@
</PropertyGroup>

<ItemGroup>
<PackageReference Include="Azure.AI.OpenAI" Version="2.0.0-beta.2" />
<PackageReference Include="Azure.AI.OpenAI" Version="2.0.0" />
<PackageReference Include="Microsoft.Extensions.Http" Version="8.0.0" />
</ItemGroup>

Expand Down
2 changes: 1 addition & 1 deletion ...ugins/BotSharp.Plugin.AzureOpenAI/Providers/Audio/AudioCompletionProvider.SpeechToText.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ private AudioTranscriptionOptions PrepareTranscriptionOptions(string? text)
var options = new AudioTranscriptionOptions
{
ResponseFormat = format,
Granularities = granularity,
TimestampGranularities = granularity,
Temperature = temperature,
Prompt = text
};
Expand Down
4 changes: 2 additions & 2 deletions ...ugins/BotSharp.Plugin.AzureOpenAI/Providers/Audio/AudioCompletionProvider.TextToSpeech.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ public async Task<BinaryData> GenerateAudioFromTextAsync(string text)
.GetAudioClient(_model);

var (voice, options) = PrepareGenerationOptions();
var result = await audioClient.GenerateSpeechFromTextAsync(text, voice, options);
var result = await audioClient.GenerateSpeechAsync(text, voice, options);
return result.Value;
}

Expand All @@ -24,7 +24,7 @@ public async Task<BinaryData> GenerateAudioFromTextAsync(string text)
var options = new SpeechGenerationOptions
{
ResponseFormat = format,
Speed = speed
SpeedRatio = speed
};

return (voice, options);
Expand Down
96 changes: 57 additions & 39 deletions src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Chat/ChatCompletionProvider.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -50,14 +50,15 @@ public async Task<RoleDialogModel> GetChatCompletions(Agent agent, List<RoleDial
var content = value.Content;
var text = content.FirstOrDefault()?.Text ?? string.Empty;

if (reason == ChatFinishReason.FunctionCall)
if (reason == ChatFinishReason.FunctionCall || reason == ChatFinishReason.ToolCalls)
{
var toolCall = value.ToolCalls.FirstOrDefault();
responseMessage = new RoleDialogModel(AgentRole.Function, text)
{
CurrentAgentId = agent.Id,
MessageId = conversations.LastOrDefault()?.MessageId ?? string.Empty,
FunctionName = value.FunctionCall.FunctionName,
FunctionArgs = value.FunctionCall.FunctionArguments
FunctionName = toolCall?.FunctionName,
FunctionArgs = toolCall?.FunctionArguments?.ToString()
};

// Somethings LLM will generate a function name with agent name.
Expand All @@ -66,17 +67,17 @@ public async Task<RoleDialogModel> GetChatCompletions(Agent agent, List<RoleDial
responseMessage.FunctionName = responseMessage.FunctionName.Split('.').Last();
}
}
else if (reason == ChatFinishReason.ToolCalls)
{
var toolCall = value.ToolCalls.FirstOrDefault();
responseMessage = new RoleDialogModel(AgentRole.Function, text)
{
CurrentAgentId = agent.Id,
MessageId = conversations.LastOrDefault()?.MessageId ?? string.Empty,
FunctionName = toolCall?.FunctionName,
FunctionArgs = toolCall?.FunctionArguments
};
}
//else if (reason == ChatFinishReason.ToolCalls)
//{
// var toolCall = value.ToolCalls.FirstOrDefault();
// responseMessage = new RoleDialogModel(AgentRole.Function, text)
// {
// CurrentAgentId = agent.Id,
// MessageId = conversations.LastOrDefault()?.MessageId ?? string.Empty,
// FunctionName = toolCall?.FunctionName,
// FunctionArgs = toolCall?.FunctionArguments
// };
//}
else
{
responseMessage = new RoleDialogModel(AgentRole.Assistant, text)
Expand Down Expand Up @@ -113,8 +114,8 @@ public async Task<RoleDialogModel> GetChatCompletions(Agent agent, List<RoleDial
Prompt = prompt,
Provider = Provider,
Model = _model,
PromptCount = value?.Usage?.InputTokens ?? 0,
CompletionCount = value?.Usage?.OutputTokens ?? 0
PromptCount = value?.Usage?.InputTokenCount ?? 0,
CompletionCount = value?.Usage?.OutputTokenCount ?? 0
});
}

Expand Down Expand Up @@ -157,20 +158,21 @@ public async Task<bool> GetChatCompletionsAsync(Agent agent,
Prompt = prompt,
Provider = Provider,
Model = _model,
PromptCount = response.Value.Usage.InputTokens,
CompletionCount = response.Value.Usage.OutputTokens
PromptCount = response.Value?.Usage?.InputTokenCount ?? 0,
CompletionCount = response.Value?.Usage?.OutputTokenCount ?? 0
});
}

if (reason == ChatFinishReason.FunctionCall)
if (reason == ChatFinishReason.FunctionCall || reason == ChatFinishReason.ToolCalls)
{
_logger.LogInformation($"[{agent.Name}]: {value.FunctionCall.FunctionName}({value.FunctionCall.FunctionArguments})");
var toolCall = value.ToolCalls?.FirstOrDefault();
_logger.LogInformation($"[{agent.Name}]: {toolCall?.FunctionName}({toolCall?.FunctionArguments})");

var funcContextIn = new RoleDialogModel(AgentRole.Function, text)
{
CurrentAgentId = agent.Id,
FunctionName = value.FunctionCall?.FunctionName,
FunctionArgs = value.FunctionCall?.FunctionArguments
FunctionName = toolCall?.FunctionName,
FunctionArgs = toolCall?.FunctionArguments?.ToString()
};

// Somethings LLM will generate a function name with agent name.
Expand Down Expand Up @@ -201,19 +203,20 @@ public async Task<bool> GetChatCompletionsStreamingAsync(Agent agent, List<RoleD

await foreach (var choice in response)
{
if (choice.FinishReason == ChatFinishReason.FunctionCall)
if (choice.FinishReason == ChatFinishReason.FunctionCall || choice.FinishReason == ChatFinishReason.ToolCalls)
{
Console.Write(choice.FunctionCallUpdate?.FunctionArgumentsUpdate);
var update = choice.ToolCallUpdates?.FirstOrDefault()?.FunctionArgumentsUpdate?.ToString() ?? string.Empty;
Console.Write(update);

await onMessageReceived(new RoleDialogModel(AgentRole.Assistant, choice.FunctionCallUpdate?.FunctionArgumentsUpdate));
await onMessageReceived(new RoleDialogModel(AgentRole.Assistant, update));
continue;
}

if (choice.ContentUpdate.IsNullOrEmpty()) continue;

_logger.LogInformation(choice.ContentUpdate[0]?.Text);

await onMessageReceived(new RoleDialogModel(choice.Role.ToString(), choice.ContentUpdate[0]?.Text ?? string.Empty));
await onMessageReceived(new RoleDialogModel(choice.Role?.ToString() ?? ChatMessageRole.Assistant.ToString(), choice.ContentUpdate[0]?.Text ?? string.Empty));
}

return true;
Expand All @@ -235,7 +238,7 @@ public async Task<bool> GetChatCompletionsStreamingAsync(Agent agent, List<RoleD
var options = new ChatCompletionOptions()
{
Temperature = temperature,
MaxTokens = maxTokens
MaxOutputTokenCount = maxTokens
};

foreach (var function in agent.Functions)
Expand Down Expand Up @@ -267,21 +270,35 @@ public async Task<bool> GetChatCompletionsStreamingAsync(Agent agent, List<RoleD
messages.Add(sample.Role == AgentRole.User ? new UserChatMessage(sample.Content) : new AssistantChatMessage(sample.Content));
}

foreach (var message in conversations)
var filteredMessages = conversations.Select(x => x).ToList();
var firstUserMsgIdx = filteredMessages.FindIndex(x => x.Role == AgentRole.User);
if (firstUserMsgIdx > 0)
{
filteredMessages = filteredMessages.Where((_, idx) => idx >= firstUserMsgIdx).ToList();
}

foreach (var message in filteredMessages)
{
if (message.Role == AgentRole.Function)
{
messages.Add(new AssistantChatMessage(string.Empty)
//messages.Add(new AssistantChatMessage(string.Empty)
//{
// FunctionCall = new ChatFunctionCall(message.FunctionName, message.FunctionArgs ?? string.Empty)
//});

//messages.Add(new FunctionChatMessage(message.FunctionName, message.Content));

messages.Add(new AssistantChatMessage(new List<ChatToolCall>
{
FunctionCall = new ChatFunctionCall(message.FunctionName, message.FunctionArgs ?? string.Empty)
});
ChatToolCall.CreateFunctionToolCall(message.FunctionName, message.FunctionName, BinaryData.FromString(message.FunctionArgs ?? string.Empty))
}));

messages.Add(new FunctionChatMessage(message.FunctionName, message.Content));
messages.Add(new ToolChatMessage(message.FunctionName, message.Content));
}
else if (message.Role == AgentRole.User)
{
var text = !string.IsNullOrWhiteSpace(message.Payload) ? message.Payload : message.Content;
var textPart = ChatMessageContentPart.CreateTextMessageContentPart(text);
var textPart = ChatMessageContentPart.CreateTextPart(text);
var contentParts = new List<ChatMessageContentPart> { textPart };

if (allowMultiModal && !message.Files.IsNullOrEmpty())
Expand All @@ -291,20 +308,20 @@ public async Task<bool> GetChatCompletionsStreamingAsync(Agent agent, List<RoleD
if (!string.IsNullOrEmpty(file.FileData))
{
var (contentType, bytes) = FileUtility.GetFileInfoFromData(file.FileData);
var contentPart = ChatMessageContentPart.CreateImageMessageContentPart(BinaryData.FromBytes(bytes), contentType, ImageChatMessageContentPartDetail.Low);
var contentPart = ChatMessageContentPart.CreateImagePart(BinaryData.FromBytes(bytes), contentType, ChatImageDetailLevel.Low);
contentParts.Add(contentPart);
}
else if (!string.IsNullOrEmpty(file.FileStorageUrl))
{
var contentType = FileUtility.GetFileContentType(file.FileStorageUrl);
var bytes = fileStorage.GetFileBytes(file.FileStorageUrl);
var contentPart = ChatMessageContentPart.CreateImageMessageContentPart(BinaryData.FromBytes(bytes), contentType, ImageChatMessageContentPartDetail.Low);
var contentPart = ChatMessageContentPart.CreateImagePart(BinaryData.FromBytes(bytes), contentType, ChatImageDetailLevel.Low);
contentParts.Add(contentPart);
}
else if (!string.IsNullOrEmpty(file.FileUrl))
{
var uri = new Uri(file.FileUrl);
var contentPart = ChatMessageContentPart.CreateImageMessageContentPart(uri, ImageChatMessageContentPartDetail.Low);
var contentPart = ChatMessageContentPart.CreateImagePart(uri, ChatImageDetailLevel.Low);
contentParts.Add(contentPart);
}
}
Expand Down Expand Up @@ -347,7 +364,7 @@ private string GetPrompt(IEnumerable<ChatMessage> messages, ChatCompletionOption
.Where(x => x as SystemChatMessage == null)
.Select(x =>
{
var fnMessage = x as FunctionChatMessage;
var fnMessage = x as ToolChatMessage;
if (fnMessage != null)
{
return $"{AgentRole.Function}: {fnMessage.Content.FirstOrDefault()?.Text ?? string.Empty}";
Expand All @@ -365,8 +382,9 @@ private string GetPrompt(IEnumerable<ChatMessage> messages, ChatCompletionOption
var assistMessage = x as AssistantChatMessage;
if (assistMessage != null)
{
return assistMessage.FunctionCall != null ?
$"{AgentRole.Assistant}: Call function {assistMessage.FunctionCall.FunctionName}({assistMessage.FunctionCall.FunctionArguments})" :
var toolCall = assistMessage.ToolCalls?.FirstOrDefault();
return toolCall != null ?
$"{AgentRole.Assistant}: Call function {toolCall?.FunctionName}({toolCall?.FunctionArguments})" :
$"{AgentRole.Assistant}: {assistMessage.Content.FirstOrDefault()?.Text ?? string.Empty}";
}

Expand Down
4 changes: 2 additions & 2 deletions src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Embedding/TextEmbeddingProvider.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ public async Task<float[]> GetVectorAsync(string text)
var options = PrepareOptions();
var response = await embeddingClient.GenerateEmbeddingAsync(text, options);
var value = response.Value;
return value.Vector.ToArray();
return value.ToFloats().ToArray();
}

public async Task<List<float[]>> GetVectorsAsync(List<string> texts)
Expand All @@ -41,7 +41,7 @@ public async Task<List<float[]>> GetVectorsAsync(List<string> texts)
var options = PrepareOptions();
var response = await embeddingClient.GenerateEmbeddingsAsync(texts, options);
var value = response.Value;
return value.Select(x => x.Vector.ToArray()).ToList();
return value.Select(x => x.ToFloats().ToArray()).ToList();
}

public void SetModelName(string model)
Expand Down
4 changes: 2 additions & 2 deletions src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/ProviderHelper.cs
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
using Azure.AI.OpenAI;
using Azure;
using System.ClientModel;

namespace BotSharp.Plugin.AzureOpenAI.Providers;

Expand All @@ -9,7 +9,7 @@ public static AzureOpenAIClient GetClient(string provider, string model, IServic
{
var settingsService = services.GetRequiredService<ILlmProviderService>();
var settings = settingsService.GetSetting(provider, model);
var client = new AzureOpenAIClient(new Uri(settings.Endpoint), new AzureKeyCredential(settings.ApiKey));
var client = new AzureOpenAIClient(new Uri(settings.Endpoint), new ApiKeyCredential(settings.ApiKey));
return client;
}

Expand Down
2 changes: 1 addition & 1 deletion src/Plugins/BotSharp.Plugin.OpenAI/BotSharp.Plugin.OpenAI.csproj
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@
</PropertyGroup>

<ItemGroup>
<PackageReference Include="OpenAI" Version="2.0.0-beta.5" />
<PackageReference Include="OpenAI" Version="2.0.0" />
<PackageReference Include="Microsoft.Extensions.Http" Version="8.0.0" />
</ItemGroup>

Expand Down
2 changes: 1 addition & 1 deletion src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/AudioCompletionProvider.SpeechToText.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ private AudioTranscriptionOptions PrepareTranscriptionOptions(string? text)
var options = new AudioTranscriptionOptions
{
ResponseFormat = format,
Granularities = granularity,
TimestampGranularities = granularity,
Temperature = temperature,
Prompt = text
};
Expand Down
4 changes: 2 additions & 2 deletions src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/AudioCompletionProvider.TextToSpeech.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ public async Task<BinaryData> GenerateAudioFromTextAsync(string text)
.GetAudioClient(_model);

var (voice, options) = PrepareGenerationOptions();
var result = await audioClient.GenerateSpeechFromTextAsync(text, voice, options);
var result = await audioClient.GenerateSpeechAsync(text, voice, options);
return result.Value;
}

Expand All @@ -24,7 +24,7 @@ public async Task<BinaryData> GenerateAudioFromTextAsync(string text)
var options = new SpeechGenerationOptions
{
ResponseFormat = format,
Speed = speed
SpeedRatio = speed
};

return (voice, options);
Expand Down
Loading

0 comments on commit 022c050

Please sign in to comment.