diff --git a/Directory.Packages.props b/Directory.Packages.props index 6f5bcd3..bf86070 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -1,69 +1,72 @@ - - - true - true - 8.6.0 - 8.2.2 - 9.0.1-preview.1.24570.5 - 0.9.6-preview - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + true + true + 8.6.0 + 8.2.2 + 9.0.1-preview.1.24570.5 + 0.9.6-preview + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/src/AppHost/AppHost.csproj b/src/AppHost/AppHost.csproj index 7dae6ee..a24e2e1 100644 --- a/src/AppHost/AppHost.csproj +++ b/src/AppHost/AppHost.csproj @@ -15,6 +15,8 @@ + + diff --git a/src/AppHost/Ollama/OllamaResource.cs b/src/AppHost/Ollama/OllamaResource.cs deleted file mode 100644 index 20ebd5c..0000000 --- a/src/AppHost/Ollama/OllamaResource.cs +++ /dev/null @@ -1,8 +0,0 @@ -namespace Aspire.Hosting; - -internal class OllamaResource(string name, string[] models, string defaultModel, bool enableGpu) : ContainerResource(name) -{ - public string[] Models { get; } = models; - public string DefaultModel { get; } = defaultModel; - public bool EnableGpu { get; } = enableGpu; -} diff --git a/src/AppHost/Ollama/OllamaResourceExtensions.cs b/src/AppHost/Ollama/OllamaResourceExtensions.cs deleted file mode 100644 index c22f708..0000000 --- a/src/AppHost/Ollama/OllamaResourceExtensions.cs +++ /dev/null @@ -1,163 +0,0 @@ -using System.Net.Http.Json; -using System.Text.Json; -using Aspire.Hosting.Lifecycle; -using Microsoft.Extensions.Logging; - -namespace Aspire.Hosting; - -internal static class OllamaResourceExtensions -{ - public static IResourceBuilder AddOllama(this IDistributedApplicationBuilder builder, string name, string[]? models = null, string? defaultModel = null, bool enableGpu = true, int? port = null) - { - const string configKey = "OllamaModel"; - defaultModel ??= builder.Configuration[configKey]; - - if (models is null or { Length: 0 }) - { - if (string.IsNullOrEmpty(defaultModel)) - { - throw new InvalidOperationException($"Expected the parameter '{nameof(defaultModel)}' or '{nameof(models)}' to be nonempty, or to find a configuration value '{configKey}', but none were provided."); - } - models = [defaultModel]; - } - - var resource = new OllamaResource(name, models, defaultModel ?? models.First(), enableGpu); - var ollama = builder.AddResource(resource) - .WithHttpEndpoint(port: port, targetPort: 11434) - .WithImage("ollama/ollama", tag: "0.3.12"); - - if (enableGpu) - { - ollama = ollama.WithContainerRuntimeArgs("--gpus=all"); - } - - builder.Services.TryAddLifecycleHook(); - - // This is a bit of a hack to show downloading models in the UI - builder.AddResource(new OllamaModelDownloaderResource($"ollama-model-downloader-{name}", resource)) - .WithInitialState(new() - { - Properties = [], - ResourceType = "ollama downloader", - State = KnownResourceStates.Hidden - }) - .ExcludeFromManifest(); - - return ollama; - } - - public static IResourceBuilder WithDataVolume(this IResourceBuilder builder) - { - return builder.WithVolume(CreateVolumeName(builder, builder.Resource.Name), "/root/.ollama"); - } - - public static IResourceBuilder WithReference(this IResourceBuilder builder, IResourceBuilder ollamaBuilder) - where TDestination : IResourceWithEnvironment - { - return builder - .WithReference(ollamaBuilder.GetEndpoint("http")) - .WithEnvironment($"{ollamaBuilder.Resource.Name}:Type", "ollama") - .WithEnvironment($"{ollamaBuilder.Resource.Name}:LlmModelName", ollamaBuilder.Resource.DefaultModel); - } - - private static string CreateVolumeName(IResourceBuilder builder, string suffix) where T : IResource - { - // Ideally this would be public - return (string)typeof(ContainerResource).Assembly - .GetType("Aspire.Hosting.Utils.VolumeNameGenerator", true)! - .GetMethod("CreateVolumeName")! - .MakeGenericMethod(typeof(T)) - .Invoke(null, [builder, suffix])!; - } - - private sealed class OllamaEnsureModelAvailableHook( - ResourceLoggerService loggerService, - ResourceNotificationService notificationService, - DistributedApplicationExecutionContext context) : IDistributedApplicationLifecycleHook - { - public Task AfterEndpointsAllocatedAsync(DistributedApplicationModel appModel, CancellationToken cancellationToken = default) - { - if (context.IsPublishMode) - { - return Task.CompletedTask; - } - - var client = new HttpClient(); - - foreach (var downloader in appModel.Resources.OfType()) - { - var ollama = downloader.ollamaResource; - - var logger = loggerService.GetLogger(downloader); - - _ = Task.Run(async () => - { - var httpEndpoint = ollama.GetEndpoint("http"); - - // TODO: Make this resilient to failure - var ollamaModelsAvailable = await client.GetFromJsonAsync($"{httpEndpoint.Url}/api/tags", new JsonSerializerOptions(JsonSerializerDefaults.Web)); - - if (ollamaModelsAvailable is null) - { - return; - } - - var availableModelNames = ollamaModelsAvailable.Models?.Select(m => m.Name) ?? []; - - var modelsToDownload = ollama.Models.Except(availableModelNames); - - if (!modelsToDownload.Any()) - { - return; - } - - logger.LogInformation("Downloading models {Models} for ollama {OllamaName}...", string.Join(", ", modelsToDownload), ollama.Name); - - await notificationService.PublishUpdateAsync(downloader, s => s with - { - State = new("Downloading models...", KnownResourceStateStyles.Info) - }); - - await Parallel.ForEachAsync(modelsToDownload, async (modelName, ct) => - { - await DownloadModelAsync(logger, httpEndpoint, modelName, ct); - }); - - await notificationService.PublishUpdateAsync(downloader, s => s with - { - State = new("Models downloaded", KnownResourceStateStyles.Success) - }); - }, - cancellationToken); - } - - return Task.CompletedTask; - } - - private static async Task DownloadModelAsync(ILogger logger, EndpointReference httpEndpoint, string? modelName, CancellationToken cancellationToken) - { - logger.LogInformation("Pulling ollama model {ModelName}...", modelName); - - var httpClient = new HttpClient { Timeout = TimeSpan.FromDays(1) }; - var request = new HttpRequestMessage(HttpMethod.Post, $"{httpEndpoint.Url}/api/pull") { Content = JsonContent.Create(new { name = modelName }) }; - var response = await httpClient.SendAsync(request, HttpCompletionOption.ResponseHeadersRead); - var responseContentStream = await response.Content.ReadAsStreamAsync(cancellationToken); - var streamReader = new StreamReader(responseContentStream); - var line = (string?)null; - while ((line = await streamReader.ReadLineAsync(cancellationToken)) is not null) - { - logger.Log(LogLevel.Information, 0, line, null, (s, ex) => s); - } - - logger.LogInformation("Finished pulling ollama mode {ModelName}", modelName); - } - - record OllamaGetTagsResponse(OllamaGetTagsResponseModel[]? Models); - record OllamaGetTagsResponseModel(string Name); - } - - private class OllamaModelDownloaderResource(string name, OllamaResource ollamaResource) : Resource(name) - { - public OllamaResource ollamaResource { get; } = ollamaResource; - } -} diff --git a/src/AppHost/Program.cs b/src/AppHost/Program.cs index e49e09d..795d4d8 100644 --- a/src/AppHost/Program.cs +++ b/src/AppHost/Program.cs @@ -1,4 +1,5 @@ -using Microsoft.Extensions.Configuration.Json; +#pragma warning disable ASPIREHOSTINGPYTHON001 +using Microsoft.Extensions.Configuration.Json; using Microsoft.Extensions.Hosting; using Projects; @@ -24,7 +25,8 @@ .GetEndpoint("https"); // Use this if you want to use Ollama -var chatCompletion = builder.AddOllama("chatcompletion").WithDataVolume(); +var ollama = builder.AddOllama("ollama").WithDataVolume().WithGPUSupport(); +var chatCompletion = ollama.AddModel("chatCompletion", "llama3.1"); // ... or use this if you want to use OpenAI (having also configured the API key in appsettings) //var chatCompletion = builder.AddConnectionString("chatcompletion"); @@ -41,11 +43,11 @@ }); } -var blobStorage = storage.AddBlobs("eshopsupport-blobs"); - -var pythonInference = builder.AddPythonUvicornApp("python-inference", - Path.Combine("..", "PythonInference"), port: 62394); - +var blobStorage = storage.AddBlobs("eshopsupport-blobs"); + +var pythonInference = builder.AddUvicornApp("python-inference", Path.Combine("..", "PythonInference"), "main:app") + .WithHttpEndpoint(env: "UVICORN_PORT", port: 62394); + var redis = builder.AddRedis("redis"); var backend = builder.AddProject("backend") diff --git a/src/AppHost/Python/PythonUvicornAppResourceBuilderExtensions.cs b/src/AppHost/Python/PythonUvicornAppResourceBuilderExtensions.cs deleted file mode 100644 index b3a2713..0000000 --- a/src/AppHost/Python/PythonUvicornAppResourceBuilderExtensions.cs +++ /dev/null @@ -1,16 +0,0 @@ -namespace Aspire.Hosting; - -public static class PythonUvicornAppResourceBuilderExtensions -{ - public static IResourceBuilder AddPythonUvicornApp(this IDistributedApplicationBuilder builder, string name, string workingDirectory, int? port = default, int? targetPort = default) - { - return builder.AddResource(new PythonUvicornAppResource(name, "python", workingDirectory)) - .WithArgs("-m", "uvicorn", "main:app") - .WithHttpEndpoint(env: "UVICORN_PORT", port: port, targetPort: targetPort); - } -} - -public class PythonUvicornAppResource(string name, string command, string workingDirectory) - : ExecutableResource(name, command, workingDirectory), IResourceWithServiceDiscovery -{ -} diff --git a/src/AppHost/appsettings.Development.json b/src/AppHost/appsettings.Development.json index a0a8b7c..901ace0 100644 --- a/src/AppHost/appsettings.Development.json +++ b/src/AppHost/appsettings.Development.json @@ -10,9 +10,6 @@ // To reduce the risk, copy this file as appsettings.Local.json and make your changes there, // since that file overrides anything configured here and will be ignored by Git. - // This is used if you're using Ollama. Be sure to pick a model that supports tools. - "OllamaModel": "llama3.1", - // This is used if you're using OpenAI "ConnectionStrings": { //"chatcompletion": "Endpoint=https://TODO.openai.azure.com/;Key=TODO;Deployment=TODO" diff --git a/src/ServiceDefaults/Clients/ChatCompletion/ServiceCollectionChatClientExtensions.cs b/src/ServiceDefaults/Clients/ChatCompletion/ServiceCollectionChatClientExtensions.cs index 31d0694..a7d33d3 100644 --- a/src/ServiceDefaults/Clients/ChatCompletion/ServiceCollectionChatClientExtensions.cs +++ b/src/ServiceDefaults/Clients/ChatCompletion/ServiceCollectionChatClientExtensions.cs @@ -12,39 +12,17 @@ public static class ServiceCollectionChatClientExtensions { public static ChatClientBuilder AddOllamaChatClient( this IHostApplicationBuilder hostBuilder, - string serviceName, - string? modelName = null) + string serviceName) { - if (modelName is null) - { - var configKey = $"{serviceName}:LlmModelName"; - modelName = hostBuilder.Configuration[configKey]; - if (string.IsNullOrEmpty(modelName)) - { - throw new InvalidOperationException($"No {nameof(modelName)} was specified, and none could be found from configuration at '{configKey}'"); - } - } + hostBuilder.AddOllamaSharpChatClient(serviceName); - return hostBuilder.Services.AddOllamaChatClient( - modelName, - new Uri($"http://{serviceName}")); - } - - public static ChatClientBuilder AddOllamaChatClient( - this IServiceCollection services, - string modelName, - Uri? uri = null) - { - uri ??= new Uri("http://localhost:11434"); - - ChatClientBuilder chatClientBuilder = services.AddChatClient(serviceProvider => { - var httpClient = serviceProvider.GetService() ?? new(); - return new OllamaChatClient(uri, modelName, httpClient); - }); + ChatClientBuilder chatClientBuilder = hostBuilder.Services.AddChatClient(static sp => + // use the IChatClient from OllamaSharp + sp.GetRequiredService()); // Temporary workaround for Ollama issues - chatClientBuilder.UsePreventStreamingWithFunctions(); - + chatClientBuilder.UsePreventStreamingWithFunctions(); + return chatClientBuilder; } diff --git a/src/ServiceDefaults/ServiceDefaults.csproj b/src/ServiceDefaults/ServiceDefaults.csproj index ab367e1..b3c9773 100644 --- a/src/ServiceDefaults/ServiceDefaults.csproj +++ b/src/ServiceDefaults/ServiceDefaults.csproj @@ -12,6 +12,8 @@ + +