diff --git a/BotSharp.sln b/BotSharp.sln index 972dc35f7..4aa709de6 100644 --- a/BotSharp.sln +++ b/BotSharp.sln @@ -143,6 +143,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BotSharp.LLM.Tests", "tests EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BotSharp.Test.RealtimeVoice", "tests\BotSharp.Test.RealtimeVoice\BotSharp.Test.RealtimeVoice.csproj", "{B067B126-88CD-4282-BEEF-7369B64423EF}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BotSharp.Plugin.ChartHandler", "src\Plugins\BotSharp.Plugin.ChartHandler\BotSharp.Plugin.ChartHandler.csproj", "{0428DEAA-E4FE-4259-A6D8-6EDD1A9D0702}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -599,6 +601,14 @@ Global {B067B126-88CD-4282-BEEF-7369B64423EF}.Release|Any CPU.Build.0 = Release|Any CPU {B067B126-88CD-4282-BEEF-7369B64423EF}.Release|x64.ActiveCfg = Release|Any CPU {B067B126-88CD-4282-BEEF-7369B64423EF}.Release|x64.Build.0 = Release|Any CPU + {0428DEAA-E4FE-4259-A6D8-6EDD1A9D0702}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {0428DEAA-E4FE-4259-A6D8-6EDD1A9D0702}.Debug|Any CPU.Build.0 = Debug|Any CPU + {0428DEAA-E4FE-4259-A6D8-6EDD1A9D0702}.Debug|x64.ActiveCfg = Debug|Any CPU + {0428DEAA-E4FE-4259-A6D8-6EDD1A9D0702}.Debug|x64.Build.0 = Debug|Any CPU + {0428DEAA-E4FE-4259-A6D8-6EDD1A9D0702}.Release|Any CPU.ActiveCfg = Release|Any CPU + {0428DEAA-E4FE-4259-A6D8-6EDD1A9D0702}.Release|Any CPU.Build.0 = Release|Any CPU + {0428DEAA-E4FE-4259-A6D8-6EDD1A9D0702}.Release|x64.ActiveCfg = Release|Any CPU + {0428DEAA-E4FE-4259-A6D8-6EDD1A9D0702}.Release|x64.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -668,6 +678,7 @@ Global {7D0DB012-9798-4BB9-B15B-A5B0B7B3B094} = {32FAFFFE-A4CB-4FEE-BF7C-84518BBC6DCC} {7C0C7D13-D161-4AB0-9C29-83A0F1FF990E} = {32FAFFFE-A4CB-4FEE-BF7C-84518BBC6DCC} {B067B126-88CD-4282-BEEF-7369B64423EF} = {32FAFFFE-A4CB-4FEE-BF7C-84518BBC6DCC} + {0428DEAA-E4FE-4259-A6D8-6EDD1A9D0702} = {51AFE054-AE99-497D-A593-69BAEFB5106F} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {A9969D89-C98B-40A5-A12B-FC87E55B3A19} diff --git a/src/Infrastructure/BotSharp.Abstraction/Messaging/BotSharpMessageParser.cs b/src/Infrastructure/BotSharp.Abstraction/Messaging/BotSharpMessageParser.cs index 1f495a440..25a97b2b9 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Messaging/BotSharpMessageParser.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Messaging/BotSharpMessageParser.cs @@ -1,7 +1,7 @@ -using BotSharp.Abstraction.Messaging.Models.RichContent.Template; -using BotSharp.Abstraction.Messaging.Models.RichContent; using System.Text.Json; using System.Reflection; +using BotSharp.Abstraction.Messaging.Models.RichContent.Template; +using BotSharp.Abstraction.Messaging.Models.RichContent; namespace BotSharp.Abstraction.Messaging; @@ -38,6 +38,10 @@ public static class BotSharpMessageParser { targetType = typeof(TextMessage); } + else if (richType == RichTypeEnum.ProgramCode) + { + targetType = typeof(ProgramCodeTemplateMessage); + } else if (richType == RichTypeEnum.GenericTemplate) { if (root.TryGetProperty("element_type", out element)) @@ -84,9 +88,9 @@ public static class BotSharpMessageParser { targetType = typeof(CouponTemplateMessage); } - else if (templateType == TemplateTypeEnum.Product) + else if (templateType == TemplateTypeEnum.ProgramCode) { - targetType = typeof(ProductTemplateMessage); + targetType = typeof(ProgramCodeTemplateMessage); } else if (templateType == TemplateTypeEnum.Generic) { diff --git a/src/Infrastructure/BotSharp.Abstraction/Messaging/Enums/RichTypeEnum.cs b/src/Infrastructure/BotSharp.Abstraction/Messaging/Enums/RichTypeEnum.cs index 7603f7906..8d66904bb 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Messaging/Enums/RichTypeEnum.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Messaging/Enums/RichTypeEnum.cs @@ -9,4 +9,5 @@ public static class RichTypeEnum public const string QuickReply = "quick_reply"; public const string Text = "text"; public const string Attachment = "attachment"; + public const string ProgramCode = "program_code"; } diff --git a/src/Infrastructure/BotSharp.Abstraction/Messaging/Enums/TemplateTypeEnum.cs b/src/Infrastructure/BotSharp.Abstraction/Messaging/Enums/TemplateTypeEnum.cs index 74f887350..73a4e9d4c 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Messaging/Enums/TemplateTypeEnum.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Messaging/Enums/TemplateTypeEnum.cs @@ -6,5 +6,5 @@ public static class TemplateTypeEnum public const string Coupon = "coupon"; public const string Generic = "generic"; public const string MultiSelect = "multi-select"; - public const string Product = "product"; + public const string ProgramCode = "program_code"; } diff --git a/src/Infrastructure/BotSharp.Abstraction/Messaging/Models/RichContent/Template/ProductTemplateMessage.cs b/src/Infrastructure/BotSharp.Abstraction/Messaging/Models/RichContent/Template/ProductTemplateMessage.cs deleted file mode 100644 index 05af097a7..000000000 --- a/src/Infrastructure/BotSharp.Abstraction/Messaging/Models/RichContent/Template/ProductTemplateMessage.cs +++ /dev/null @@ -1,19 +0,0 @@ -namespace BotSharp.Abstraction.Messaging.Models.RichContent.Template; - -public class ProductTemplateMessage : IRichMessage, ITemplateMessage -{ - [JsonPropertyName("rich_type")] - public string RichType => RichTypeEnum.GenericTemplate; - - [JsonPropertyName("text")] - [Translate] - public string Text { get; set; } = string.Empty; - - [JsonPropertyName("template_type")] - public string TemplateType => TemplateTypeEnum.Product; -} - -public class ProductElement -{ - public string Id { get; set; } -} diff --git a/src/Infrastructure/BotSharp.Abstraction/Messaging/Models/RichContent/Template/ProgramCodeTemplateMessage.cs b/src/Infrastructure/BotSharp.Abstraction/Messaging/Models/RichContent/Template/ProgramCodeTemplateMessage.cs new file mode 100644 index 000000000..60f6464ad --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Messaging/Models/RichContent/Template/ProgramCodeTemplateMessage.cs @@ -0,0 +1,16 @@ +namespace BotSharp.Abstraction.Messaging.Models.RichContent.Template; + +public class ProgramCodeTemplateMessage : IRichMessage, ITemplateMessage +{ + [JsonPropertyName("rich_type")] + public string RichType => RichTypeEnum.ProgramCode; + + [JsonPropertyName("text")] + public string Text { get; set; } = string.Empty; + + [JsonPropertyName("template_type")] + public virtual string TemplateType { get; set; } = TemplateTypeEnum.ProgramCode; + + [JsonPropertyName("language")] + public string Language { get; set; } = string.Empty; +} diff --git a/src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj b/src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj index ddcf68cb4..635d35d74 100644 --- a/src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj +++ b/src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj @@ -86,7 +86,6 @@ - @@ -97,6 +96,7 @@ + @@ -188,7 +188,7 @@ PreserveNewest - + PreserveNewest diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.SelectFile.cs b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.SelectFile.cs index 13525fb13..e00f9158e 100644 --- a/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.SelectFile.cs +++ b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.SelectFile.cs @@ -65,7 +65,7 @@ private async Task> SelectFiles(IEnumerable + + + net8.0 + enable + enable + + + + + + + + + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + + + + + diff --git a/src/Plugins/BotSharp.Plugin.ChartHandler/ChartHandlerPlugin.cs b/src/Plugins/BotSharp.Plugin.ChartHandler/ChartHandlerPlugin.cs new file mode 100644 index 000000000..1161f6e4b --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.ChartHandler/ChartHandlerPlugin.cs @@ -0,0 +1,18 @@ +using Microsoft.Extensions.Configuration; + +namespace BotSharp.Plugin.ChartHandler; + +public class ChartHandlerPlugin : IBotSharpPlugin +{ + public string Id => "9dacac1d-2e29-4f01-9d66-b0201f05a9fa"; + public string Name => "Chart Plotter"; + public string Description => "AI plots chart"; + public string IconUrl => "https://cdn-icons-png.flaticon.com/512/423/423786.png"; + public string[] AgentIds => []; + + public void RegisterDI(IServiceCollection services, IConfiguration config) + { + services.AddScoped(); + } + +} diff --git a/src/Plugins/BotSharp.Plugin.ChartHandler/Enums/UtilityName.cs b/src/Plugins/BotSharp.Plugin.ChartHandler/Enums/UtilityName.cs new file mode 100644 index 000000000..ee97a4a5c --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.ChartHandler/Enums/UtilityName.cs @@ -0,0 +1,6 @@ +namespace BotSharp.Plugin.ChartHandler.Enums; + +public class UtilityName +{ + public const string ChartPlotter = "chart-plotter"; +} diff --git a/src/Plugins/BotSharp.Plugin.ChartHandler/Functions/PlotChartFn.cs b/src/Plugins/BotSharp.Plugin.ChartHandler/Functions/PlotChartFn.cs new file mode 100644 index 000000000..ad8de0d79 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.ChartHandler/Functions/PlotChartFn.cs @@ -0,0 +1,82 @@ +using BotSharp.Abstraction.Messaging.Models.RichContent.Template; + +namespace BotSharp.Plugin.ChartHandler.Functions; + +public class PlotChartFn : IFunctionCallback +{ + private readonly IServiceProvider _services; + private readonly ILogger _logger; + + public string Name => "util-chart-plot_chart"; + public string Indication => "Plotting chart"; + + + public PlotChartFn( + IServiceProvider services, + ILogger logger) + { + _services = services; + _logger = logger; + } + + public async Task Execute(RoleDialogModel message) + { + var db = _services.GetRequiredService(); + var agentService = _services.GetRequiredService(); + var convService = _services.GetRequiredService(); + + var args = JsonSerializer.Deserialize(message.FunctionArgs); + var agent = await agentService.GetAgent(message.CurrentAgentId); + var inst = db.GetAgentTemplate(BuiltInAgentId.UtilityAssistant, "util-chart-plot_instruction"); + var innerAgent = new Agent + { + Id = agent.Id, + Name = agent.Name, + Instruction = inst, + TemplateDict = new Dictionary + { + { "plotting_requirement", args?.PlottingRequirement ?? string.Empty }, + { "chart_element_id", $"chart-{message.MessageId}" } + } + }; + + var response = await GetChatCompletion(innerAgent, [ + new RoleDialogModel(AgentRole.User, "Please follow the instruction to generate the javascript code.") + { + CurrentAgentId = message.CurrentAgentId, + MessageId = message.MessageId + } + ]); + + var obj = response.JsonContent(); + message.Content = obj?.GreetingMessage ?? "Here is the chart you ask for:"; + message.RichContent = new RichContent + { + Recipient = new Recipient { Id = convService.ConversationId }, + Message = new ProgramCodeTemplateMessage + { + Text = obj?.JsCode ?? string.Empty, + Language = "javascript" + } + }; + message.StopCompletion = true; + return true; + } + + private async Task GetChatCompletion(Agent agent, List dialogs) + { + try + { + var llmProviderService = _services.GetRequiredService(); + var completion = CompletionProvider.GetChatCompletion(_services, provider: "openai", model: "gpt-4.1"); + var response = await completion.GetChatCompletions(agent, dialogs); + return response.Content; + } + catch (Exception ex) + { + var error = $"Error when plotting chart. {ex.Message}"; + _logger.LogWarning(ex, error); + return error; + } + } +} diff --git a/src/Plugins/BotSharp.Plugin.ChartHandler/Hooks/ChartHandlerUtilityHook.cs b/src/Plugins/BotSharp.Plugin.ChartHandler/Hooks/ChartHandlerUtilityHook.cs new file mode 100644 index 000000000..02793ffe3 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.ChartHandler/Hooks/ChartHandlerUtilityHook.cs @@ -0,0 +1,24 @@ +namespace BotSharp.Plugin.ChartHandler.Hooks; + +public class ChartHandlerUtilityHook : IAgentUtilityHook +{ + private const string PLOT_CHART_FN = "util-chart-plot_chart"; + + public void AddUtilities(List utilities) + { + var item = new AgentUtility + { + Category = "chart", + Name = UtilityName.ChartPlotter, + Items = [ + new UtilityItem + { + FunctionName = PLOT_CHART_FN, + TemplateName = $"{PLOT_CHART_FN}.fn" + } + ] + }; + + utilities.Add(item); + } +} diff --git a/src/Plugins/BotSharp.Plugin.ChartHandler/LlmContext/LlmContextIn.cs b/src/Plugins/BotSharp.Plugin.ChartHandler/LlmContext/LlmContextIn.cs new file mode 100644 index 000000000..daaef2712 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.ChartHandler/LlmContext/LlmContextIn.cs @@ -0,0 +1,10 @@ +using System.Text.Json.Serialization; + +namespace BotSharp.Plugin.ChartHandler.LlmContext; + +public class LlmContextIn +{ + [JsonPropertyName("plotting_requirement")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? PlottingRequirement { get; set; } +} diff --git a/src/Plugins/BotSharp.Plugin.ChartHandler/LlmContext/LlmContextOut.cs b/src/Plugins/BotSharp.Plugin.ChartHandler/LlmContext/LlmContextOut.cs new file mode 100644 index 000000000..7109f72d7 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.ChartHandler/LlmContext/LlmContextOut.cs @@ -0,0 +1,14 @@ +using System.Text.Json.Serialization; + +namespace BotSharp.Plugin.ChartHandler.LlmContext; + +public class LlmContextOut +{ + [JsonPropertyName("greeting_message")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? GreetingMessage { get; set; } + + [JsonPropertyName("js_code")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? JsCode { get; set; } +} diff --git a/src/Plugins/BotSharp.Plugin.ChartHandler/Using.cs b/src/Plugins/BotSharp.Plugin.ChartHandler/Using.cs new file mode 100644 index 000000000..fa94465cd --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.ChartHandler/Using.cs @@ -0,0 +1,32 @@ +global using System; +global using System.Collections.Generic; +global using System.Text; +global using System.Linq; +global using System.Text.Json; +global using System.Net.Mime; +global using System.Threading.Tasks; +global using Microsoft.Extensions.DependencyInjection; +global using Microsoft.Extensions.Logging; +global using BotSharp.Abstraction.Agents; +global using BotSharp.Abstraction.Conversations; +global using BotSharp.Abstraction.Plugins; +global using BotSharp.Abstraction.Conversations.Models; +global using BotSharp.Abstraction.Functions; +global using BotSharp.Abstraction.Agents.Models; +global using BotSharp.Abstraction.Agents.Enums; +global using BotSharp.Abstraction.Files.Enums; +global using BotSharp.Abstraction.Files.Models; +global using BotSharp.Abstraction.Files; +global using BotSharp.Abstraction.MLTasks; +global using BotSharp.Abstraction.Utilities; +global using BotSharp.Abstraction.Agents.Settings; +global using BotSharp.Abstraction.Functions.Models; +global using BotSharp.Abstraction.Repositories; +global using BotSharp.Abstraction.Settings; +global using BotSharp.Abstraction.Messaging; +global using BotSharp.Abstraction.Messaging.Models.RichContent; +global using BotSharp.Abstraction.Options; +global using BotSharp.Core.Infrastructures; +global using BotSharp.Plugin.ChartHandler.Enums; +global using BotSharp.Plugin.ChartHandler.LlmContext; +global using BotSharp.Plugin.ChartHandler.Hooks; \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.ChartHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-chart-plot_chart.json b/src/Plugins/BotSharp.Plugin.ChartHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-chart-plot_chart.json new file mode 100644 index 000000000..da950832a --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.ChartHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-chart-plot_chart.json @@ -0,0 +1,14 @@ +{ + "name": "util-chart-plot_chart", + "description": "If the user requests you plotting or generating charts, you can call this function to generate charts in any format that user requested.", + "parameters": { + "type": "object", + "properties": { + "plotting_requirement": { + "type": "string", + "description": "The requirement that user posted for plotting or generating charts." + } + }, + "required": [ "plotting_requirement" ] + } +} \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.ChartHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-chart-plot_chart.fn.liquid b/src/Plugins/BotSharp.Plugin.ChartHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-chart-plot_chart.fn.liquid new file mode 100644 index 000000000..e54b3c4a4 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.ChartHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-chart-plot_chart.fn.liquid @@ -0,0 +1 @@ +Please call function util-chart-plot_chart if user wants to plot or generate charts. \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.ChartHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-chart-plot_instruction.liquid b/src/Plugins/BotSharp.Plugin.ChartHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-chart-plot_instruction.liquid new file mode 100644 index 000000000..9bffa73f0 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.ChartHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-chart-plot_instruction.liquid @@ -0,0 +1,22 @@ +Please take a look at "Plotting Requirement" and generate a javascript code that can be used to render the charts on an html element. + + +=== Plotting Requirement === +{{ plotting_requirement }} + + +***** Important ***** +** Your output must be a single block with everything needed inside. +** You need to import Plotly.js to plot the charts. The script source should be "https://cdn.plot.ly/plotly-3.0.1.min.js". +** You need to add the MODE bar for each chart you plot. +** You must render the charts on the div html element with id {{ chart_element_id }}. +** You must not create any new html element. +** You must not apply any styles on any html element. +** You must generate as less token as possible. + +*** Response Format *** +You must output the response in the following JSON format: +{ + "greeting_message": "A short polite message that informs user that the charts have been generated.", + "js_code": "The javascript code that can generate the charts as requested." +} \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.FileHandler/BotSharp.Plugin.FileHandler.csproj b/src/Plugins/BotSharp.Plugin.FileHandler/BotSharp.Plugin.FileHandler.csproj index 2147d178c..2b343db0f 100644 --- a/src/Plugins/BotSharp.Plugin.FileHandler/BotSharp.Plugin.FileHandler.csproj +++ b/src/Plugins/BotSharp.Plugin.FileHandler/BotSharp.Plugin.FileHandler.csproj @@ -1,4 +1,4 @@ - + $(TargetFramework) @@ -9,10 +9,6 @@ $(GenerateDocumentationFile) $(SolutionDir)packages - - - - diff --git a/src/Plugins/BotSharp.Plugin.FileHandler/LlmContexts/LlmContextIn.cs b/src/Plugins/BotSharp.Plugin.FileHandler/LlmContexts/LlmContextIn.cs index a35e83bd2..9280bfd71 100644 --- a/src/Plugins/BotSharp.Plugin.FileHandler/LlmContexts/LlmContextIn.cs +++ b/src/Plugins/BotSharp.Plugin.FileHandler/LlmContexts/LlmContextIn.cs @@ -12,10 +12,6 @@ public class LlmContextIn [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] public string? ImageDescription { get; set; } - //[JsonPropertyName("image_url")] - //[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] - //public string? ImageUrl { get; set; } - [JsonPropertyName("image_urls")] [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] public IEnumerable? ImageUrls { get; set; } diff --git a/src/WebStarter/WebStarter.csproj b/src/WebStarter/WebStarter.csproj index 5db5199fe..2610d4f11 100644 --- a/src/WebStarter/WebStarter.csproj +++ b/src/WebStarter/WebStarter.csproj @@ -36,11 +36,11 @@ - + @@ -75,6 +75,7 @@ + diff --git a/src/WebStarter/appsettings.json b/src/WebStarter/appsettings.json index 2b217318a..4e7a42633 100644 --- a/src/WebStarter/appsettings.json +++ b/src/WebStarter/appsettings.json @@ -539,6 +539,7 @@ "BotSharp.Plugin.FileHandler", "BotSharp.Plugin.EmailHandler", "BotSharp.Plugin.AudioHandler", + "BotSharp.Plugin.ChartHandler", "BotSharp.Plugin.TencentCos" ], "ExcludedFunctions": [