diff --git a/src/Libraries/.editorconfig b/src/Libraries/.editorconfig index d33ed319482..90ede887742 100644 --- a/src/Libraries/.editorconfig +++ b/src/Libraries/.editorconfig @@ -2936,7 +2936,7 @@ dotnet_diagnostic.S103.severity = suggestion # Title : Files should not have too many lines of code # Category : Major Code Smell # Help Link: https://rules.sonarsource.com/csharp/RSPEC-104 -dotnet_diagnostic.S104.severity = warning +dotnet_diagnostic.S104.severity = none # Title : Finalizers should not throw exceptions # Category : Blocker Bug diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/CHANGELOG.md b/src/Libraries/Microsoft.Extensions.AI.Abstractions/CHANGELOG.md index 433978c322a..97d8e9b5a29 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/CHANGELOG.md +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/CHANGELOG.md @@ -5,6 +5,7 @@ - Added protected copy constructors to options types (e.g. `ChatOptions`). - Fixed `EmbeddingGeneratorOptions`/`SpeechToTextOptions` `Clone` methods to correctly copy all properties. - Fixed `ToChatResponse` to not overwrite `ChatMessage/ChatResponse.CreatedAt` with older timestamps during coalescing. +- Added `[Experimental]` support for background responses, such that non-streaming responses are allowed to be pollable, and such that responses and response updates can be tagged with continuation tokens to support later resumption. ## 9.9.1 diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/ChatCompletion/ChatOptions.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/ChatCompletion/ChatOptions.cs index 4447bace386..1415ae68055 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/ChatCompletion/ChatOptions.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/ChatCompletion/ChatOptions.cs @@ -3,6 +3,7 @@ using System; using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; using System.Text.Json.Serialization; namespace Microsoft.Extensions.AI; @@ -25,8 +26,10 @@ protected ChatOptions(ChatOptions? other) } AdditionalProperties = other.AdditionalProperties?.Clone(); + AllowBackgroundResponses = other.AllowBackgroundResponses; AllowMultipleToolCalls = other.AllowMultipleToolCalls; ConversationId = other.ConversationId; + ContinuationToken = other.ContinuationToken; FrequencyPenalty = other.FrequencyPenalty; Instructions = other.Instructions; MaxOutputTokens = other.MaxOutputTokens; @@ -155,6 +158,47 @@ protected ChatOptions(ChatOptions? other) [JsonIgnore] public IList? Tools { get; set; } + /// Gets or sets a value indicating whether the background responses are allowed. + /// + /// + /// Background responses allow running long-running operations or tasks asynchronously in the background that can be resumed by streaming APIs + /// and polled for completion by non-streaming APIs. + /// + /// + /// When this property is set to true, non-streaming APIs may start a background operation and return an initial + /// response with a continuation token. Subsequent calls to the same API should be made in a polling manner with + /// the continuation token to get the final result of the operation. + /// + /// + /// When this property is set to true, streaming APIs may also start a background operation and begin streaming + /// response updates until the operation is completed. If the streaming connection is interrupted, the + /// continuation token obtained from the last update that has one should be supplied to a subsequent call to the same streaming API + /// to resume the stream from the point of interruption and continue receiving updates until the operation is completed. + /// + /// + /// This property only takes effect if the implementation it's used with supports background responses. + /// If the implementation does not support background responses, this property will be ignored. + /// + /// + [Experimental("MEAI001")] + [JsonIgnore] + public bool? AllowBackgroundResponses { get; set; } + + /// Gets or sets the continuation token for resuming and getting the result of the chat response identified by this token. + /// + /// This property is used for background responses that can be activated via the + /// property if the implementation supports them. + /// Streamed background responses, such as those returned by default by , + /// can be resumed if interrupted. This means that a continuation token obtained from the + /// of an update just before the interruption occurred can be passed to this property to resume the stream from the point of interruption. + /// Non-streamed background responses, such as those returned by , + /// can be polled for completion by obtaining the token from the property + /// and passing it to this property on subsequent calls to . + /// + [Experimental("MEAI001")] + [JsonIgnore] + public object? ContinuationToken { get; set; } + /// /// Gets or sets a callback responsible for creating the raw representation of the chat options from an underlying implementation. /// diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/ChatCompletion/ChatResponse.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/ChatCompletion/ChatResponse.cs index 0889fed17d6..ea513d2f073 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/ChatCompletion/ChatResponse.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/ChatCompletion/ChatResponse.cs @@ -88,6 +88,23 @@ public IList Messages /// Gets or sets usage details for the chat response. public UsageDetails? Usage { get; set; } + /// Gets or sets the continuation token for getting result of the background chat response. + /// + /// implementations that support background responses will return + /// a continuation token if background responses are allowed in + /// and the result of the response has not been obtained yet. If the response has completed and the result has been obtained, + /// the token will be . + /// + /// This property should be used in conjunction with to + /// continue to poll for the completion of the response. Pass this token to + /// on subsequent calls to + /// to poll for completion. + /// + /// + [Experimental("MEAI001")] + [JsonIgnore] + public object? ContinuationToken { get; set; } + /// Gets or sets the raw representation of the chat response from an underlying implementation. /// /// If a is created to represent some underlying object from another object @@ -143,6 +160,7 @@ public ChatResponseUpdate[] ToChatResponseUpdates() ResponseId = ResponseId, CreatedAt = message.CreatedAt ?? CreatedAt, + ContinuationToken = ContinuationToken, }; } diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/ChatCompletion/ChatResponseUpdate.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/ChatCompletion/ChatResponseUpdate.cs index 0605d0785bb..8b24b5c6b19 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/ChatCompletion/ChatResponseUpdate.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/ChatCompletion/ChatResponseUpdate.cs @@ -136,6 +136,21 @@ public IList Contents /// public override string ToString() => Text; + /// Gets or sets the continuation token for resuming the streamed chat response of which this update is a part. + /// + /// implementations that support background responses will return + /// a continuation token on each update if background responses are allowed in + /// except of the last update, for which the token will be . + /// + /// This property should be used for stream resumption, where the continuation token of the latest received update should be + /// passed to on subsequent calls to + /// to resume streaming from the point of interruption. + /// + /// + [Experimental("MEAI001")] + [JsonIgnore] + public object? ContinuationToken { get; set; } + /// Gets a object to display in the debugger display. [DebuggerBrowsable(DebuggerBrowsableState.Never)] private AIContent? ContentForDebuggerDisplay diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/ResponseContinuationToken.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/ResponseContinuationToken.cs new file mode 100644 index 00000000000..cf73130be10 --- /dev/null +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/ResponseContinuationToken.cs @@ -0,0 +1,68 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.ComponentModel; +using System.Diagnostics.CodeAnalysis; +using System.Text.Json; +using System.Text.Json.Serialization; +using Microsoft.Shared.Diagnostics; + +namespace Microsoft.Extensions.AI; + +/// +/// Represents a token used to resume, continue, or rehydrate an operation across multiple scenarios/calls, +/// such as resuming a streamed response from a specific point or retrieving the result of a background operation. +/// Subclasses of this class encapsulate all necessary information within the token to facilitate these actions. +/// +[JsonConverter(typeof(Converter))] +[Experimental("MEAI001")] +public class ResponseContinuationToken +{ + /// Bytes representing this token. + private readonly ReadOnlyMemory _bytes; + + /// Initializes a new instance of the class. + protected ResponseContinuationToken() + { + } + + /// Initializes a new instance of the class. + /// Bytes to create the token from. + protected ResponseContinuationToken(ReadOnlyMemory bytes) + { + _bytes = bytes; + } + + /// Create a new instance of from the provided . + /// + /// Bytes representing the . + /// A equivalent to the one from which + /// the original bytes were obtained. + public static ResponseContinuationToken FromBytes(ReadOnlyMemory bytes) => new(bytes); + + /// Gets the bytes representing this . + /// Bytes representing the ."/> + public virtual ReadOnlyMemory ToBytes() => _bytes; + + /// Provides a for serializing instances. + [EditorBrowsable(EditorBrowsableState.Never)] + [Experimental("MEAI001")] + public sealed class Converter : JsonConverter + { + /// + public override ResponseContinuationToken Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) + { + return ResponseContinuationToken.FromBytes(reader.GetBytesFromBase64()); + } + + /// + public override void Write(Utf8JsonWriter writer, ResponseContinuationToken value, JsonSerializerOptions options) + { + _ = Throw.IfNull(writer); + _ = Throw.IfNull(value); + + writer.WriteBase64StringValue(value.ToBytes().Span); + } + } +} diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Utilities/AIJsonUtilities.Defaults.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Utilities/AIJsonUtilities.Defaults.cs index 721a08418bf..f1a6f7b0a1e 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Utilities/AIJsonUtilities.Defaults.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Utilities/AIJsonUtilities.Defaults.cs @@ -135,7 +135,7 @@ private static JsonSerializerOptions CreateDefaultOptions() [JsonSerializable(typeof(McpServerToolResultContent))] [JsonSerializable(typeof(McpServerToolApprovalRequestContent))] [JsonSerializable(typeof(McpServerToolApprovalResponseContent))] - + [JsonSerializable(typeof(ResponseContinuationToken))] [EditorBrowsable(EditorBrowsableState.Never)] // Never use JsonContext directly, use DefaultOptions instead. private sealed partial class JsonContext : JsonSerializerContext; diff --git a/src/Libraries/Microsoft.Extensions.AI.OpenAI/MicrosoftExtensionsAIResponsesExtensions.cs b/src/Libraries/Microsoft.Extensions.AI.OpenAI/MicrosoftExtensionsAIResponsesExtensions.cs index c632f3c45c7..0d119a742e2 100644 --- a/src/Libraries/Microsoft.Extensions.AI.OpenAI/MicrosoftExtensionsAIResponsesExtensions.cs +++ b/src/Libraries/Microsoft.Extensions.AI.OpenAI/MicrosoftExtensionsAIResponsesExtensions.cs @@ -64,7 +64,7 @@ public static ChatResponse AsChatResponse(this OpenAIResponse response, Response /// is . public static IAsyncEnumerable AsChatResponseUpdatesAsync( this IAsyncEnumerable responseUpdates, ResponseCreationOptions? options = null, CancellationToken cancellationToken = default) => - OpenAIResponsesChatClient.FromOpenAIStreamingResponseUpdatesAsync(Throw.IfNull(responseUpdates), options, cancellationToken); + OpenAIResponsesChatClient.FromOpenAIStreamingResponseUpdatesAsync(Throw.IfNull(responseUpdates), options, cancellationToken: cancellationToken); /// Creates an OpenAI from a . /// The response to convert. diff --git a/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIResponsesChatClient.cs b/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIResponsesChatClient.cs index 5da26a435ff..b1e24461f84 100644 --- a/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIResponsesChatClient.cs +++ b/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIResponsesChatClient.cs @@ -84,9 +84,18 @@ public async Task GetResponseAsync( _ = Throw.IfNull(messages); // Convert the inputs into what OpenAIResponseClient expects. - var openAIResponseItems = ToOpenAIResponseItems(messages, options); var openAIOptions = ToOpenAIResponseCreationOptions(options); + // Provided continuation token signals that an existing background response should be fetched. + if (GetContinuationToken(messages, options) is { } token) + { + var response = await _responseClient.GetResponseAsync(token.ResponseId, cancellationToken).ConfigureAwait(false); + + return FromOpenAIResponse(response, openAIOptions); + } + + var openAIResponseItems = ToOpenAIResponseItems(messages, options); + // Make the call to the OpenAIResponseClient. var task = _createResponseAsync is not null ? _createResponseAsync(_responseClient, openAIResponseItems, openAIOptions, cancellationToken.ToRequestOptions(streaming: false)) : @@ -104,6 +113,7 @@ internal static ChatResponse FromOpenAIResponse(OpenAIResponse openAIResponse, R { ConversationId = openAIOptions?.StoredOutputEnabled is false ? null : openAIResponse.Id, CreatedAt = openAIResponse.CreatedAt, + ContinuationToken = CreateContinuationToken(openAIResponse), FinishReason = ToFinishReason(openAIResponse.IncompleteStatusDetails?.Reason), ModelId = openAIResponse.Model, RawRepresentation = openAIResponse, @@ -217,26 +227,39 @@ public IAsyncEnumerable GetStreamingResponseAsync( { _ = Throw.IfNull(messages); - var openAIResponseItems = ToOpenAIResponseItems(messages, options); var openAIOptions = ToOpenAIResponseCreationOptions(options); + // Provided continuation token signals that an existing background response should be fetched. + if (GetContinuationToken(messages, options) is { } token) + { + IAsyncEnumerable updates = _responseClient.GetResponseStreamingAsync(token.ResponseId, token.SequenceNumber, cancellationToken); + + return FromOpenAIStreamingResponseUpdatesAsync(updates, openAIOptions, token.ResponseId, cancellationToken); + } + + var openAIResponseItems = ToOpenAIResponseItems(messages, options); + var streamingUpdates = _createResponseStreamingAsync is not null ? _createResponseStreamingAsync(_responseClient, openAIResponseItems, openAIOptions, cancellationToken.ToRequestOptions(streaming: true)) : _responseClient.CreateResponseStreamingAsync(openAIResponseItems, openAIOptions, cancellationToken); - return FromOpenAIStreamingResponseUpdatesAsync(streamingUpdates, openAIOptions, cancellationToken); + return FromOpenAIStreamingResponseUpdatesAsync(streamingUpdates, openAIOptions, cancellationToken: cancellationToken); } internal static async IAsyncEnumerable FromOpenAIStreamingResponseUpdatesAsync( - IAsyncEnumerable streamingResponseUpdates, ResponseCreationOptions? options, [EnumeratorCancellation] CancellationToken cancellationToken = default) + IAsyncEnumerable streamingResponseUpdates, + ResponseCreationOptions? options, + string? resumeResponseId = null, + [EnumeratorCancellation] CancellationToken cancellationToken = default) { DateTimeOffset? createdAt = null; - string? responseId = null; - string? conversationId = null; + string? responseId = resumeResponseId; + string? conversationId = options?.StoredOutputEnabled is false ? null : resumeResponseId; string? modelId = null; string? lastMessageId = null; ChatRole? lastRole = null; bool anyFunctions = false; + ResponseStatus? latestResponseStatus = null; await foreach (var streamingUpdate in streamingResponseUpdates.WithCancellation(cancellationToken).ConfigureAwait(false)) { @@ -250,6 +273,11 @@ ChatResponseUpdate CreateUpdate(AIContent? content = null) => ModelId = modelId, RawRepresentation = streamingUpdate, ResponseId = responseId, + ContinuationToken = CreateContinuationToken( + responseId!, + latestResponseStatus, + options?.BackgroundModeEnabled, + streamingUpdate.SequenceNumber) }; switch (streamingUpdate) @@ -259,10 +287,48 @@ ChatResponseUpdate CreateUpdate(AIContent? content = null) => responseId = createdUpdate.Response.Id; conversationId = options?.StoredOutputEnabled is false ? null : responseId; modelId = createdUpdate.Response.Model; + latestResponseStatus = createdUpdate.Response.Status; + goto default; + + case StreamingResponseQueuedUpdate queuedUpdate: + createdAt = queuedUpdate.Response.CreatedAt; + responseId = queuedUpdate.Response.Id; + conversationId = options?.StoredOutputEnabled is false ? null : responseId; + modelId = queuedUpdate.Response.Model; + latestResponseStatus = queuedUpdate.Response.Status; + goto default; + + case StreamingResponseInProgressUpdate inProgressUpdate: + createdAt = inProgressUpdate.Response.CreatedAt; + responseId = inProgressUpdate.Response.Id; + conversationId = options?.StoredOutputEnabled is false ? null : responseId; + modelId = inProgressUpdate.Response.Model; + latestResponseStatus = inProgressUpdate.Response.Status; + goto default; + + case StreamingResponseIncompleteUpdate incompleteUpdate: + createdAt = incompleteUpdate.Response.CreatedAt; + responseId = incompleteUpdate.Response.Id; + conversationId = options?.StoredOutputEnabled is false ? null : responseId; + modelId = incompleteUpdate.Response.Model; + latestResponseStatus = incompleteUpdate.Response.Status; + goto default; + + case StreamingResponseFailedUpdate failedUpdate: + createdAt = failedUpdate.Response.CreatedAt; + responseId = failedUpdate.Response.Id; + conversationId = options?.StoredOutputEnabled is false ? null : responseId; + modelId = failedUpdate.Response.Model; + latestResponseStatus = failedUpdate.Response.Status; goto default; case StreamingResponseCompletedUpdate completedUpdate: { + createdAt = completedUpdate.Response.CreatedAt; + responseId = completedUpdate.Response.Id; + conversationId = options?.StoredOutputEnabled is false ? null : responseId; + modelId = completedUpdate.Response.Model; + latestResponseStatus = completedUpdate.Response?.Status; var update = CreateUpdate(ToUsageDetails(completedUpdate.Response) is { } usage ? new UsageContent(usage) : null); update.FinishReason = ToFinishReason(completedUpdate.Response?.IncompleteStatusDetails?.Reason) ?? @@ -412,6 +478,7 @@ private ResponseCreationOptions ToOpenAIResponseCreationOptions(ChatOptions? opt result.PreviousResponseId ??= options.ConversationId; result.Temperature ??= options.Temperature; result.TopP ??= options.TopP; + result.BackgroundModeEnabled ??= options.AllowBackgroundResponses; if (options.Instructions is { } instructions) { @@ -892,6 +959,59 @@ private static void AddAllMcpFilters(IList toolNames, McpToolFilter filt } } + private static OpenAIResponsesContinuationToken? CreateContinuationToken(OpenAIResponse openAIResponse) + { + return CreateContinuationToken( + responseId: openAIResponse.Id, + responseStatus: openAIResponse.Status, + isBackgroundModeEnabled: openAIResponse.BackgroundModeEnabled); + } + + private static OpenAIResponsesContinuationToken? CreateContinuationToken( + string responseId, + ResponseStatus? responseStatus, + bool? isBackgroundModeEnabled, + int? updateSequenceNumber = null) + { + if (isBackgroundModeEnabled is not true) + { + return null; + } + + // Returns a continuation token for in-progress or queued responses as they are not yet complete. + // Also returns a continuation token if there is no status but there is a sequence number, + // which can occur for certain streaming updates related to response content part updates: response.content_part.*, + // response.output_text.* + if ((responseStatus is ResponseStatus.InProgress or ResponseStatus.Queued) || + (responseStatus is null && updateSequenceNumber is not null)) + { + return new OpenAIResponsesContinuationToken(responseId) + { + SequenceNumber = updateSequenceNumber, + }; + } + + // For all other statuses: completed, failed, canceled, incomplete + // return null to indicate the operation is finished allowing the caller + // to stop and access the final result, failure details, reason for incompletion, etc. + return null; + } + + private static OpenAIResponsesContinuationToken? GetContinuationToken(IEnumerable messages, ChatOptions? options = null) + { + if (options?.ContinuationToken is { } token) + { + if (messages.Any()) + { + throw new InvalidOperationException("Messages are not allowed when continuing a background response using a continuation token."); + } + + return OpenAIResponsesContinuationToken.FromToken(token); + } + + return null; + } + /// Provides an wrapper for a . internal sealed class ResponseToolAITool(ResponseTool tool) : AITool { diff --git a/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIResponsesContinuationToken.cs b/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIResponsesContinuationToken.cs new file mode 100644 index 00000000000..229f8b40f69 --- /dev/null +++ b/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIResponsesContinuationToken.cs @@ -0,0 +1,117 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.IO; +using System.Text.Json; +using Microsoft.Shared.Diagnostics; + +namespace Microsoft.Extensions.AI; + +/// Represents a continuation token for OpenAI responses. +/// +/// The token is used for resuming streamed background responses and continuing +/// non-streamed background responses until completion. +/// +internal sealed class OpenAIResponsesContinuationToken : ResponseContinuationToken +{ + /// Initializes a new instance of the class. + internal OpenAIResponsesContinuationToken(string responseId) + { + ResponseId = responseId; + } + + /// Gets the Id of the response. + internal string ResponseId { get; } + + /// Gets or sets the sequence number of a streamed update. + internal int? SequenceNumber { get; set; } + + /// + public override ReadOnlyMemory ToBytes() + { + using MemoryStream stream = new(); + using Utf8JsonWriter writer = new(stream); + + writer.WriteStartObject(); + + writer.WriteString("responseId", ResponseId); + + if (SequenceNumber.HasValue) + { + writer.WriteNumber("sequenceNumber", SequenceNumber.Value); + } + + writer.WriteEndObject(); + + writer.Flush(); + + return stream.ToArray(); + } + + /// Create a new instance of from the provided . + /// + /// The token to create the from. + /// A equivalent of the provided . + internal static OpenAIResponsesContinuationToken FromToken(object token) + { + if (token is OpenAIResponsesContinuationToken openAIResponsesContinuationToken) + { + return openAIResponsesContinuationToken; + } + + if (token is not ResponseContinuationToken) + { + Throw.ArgumentException(nameof(token), "Failed to create OpenAIResponsesResumptionToken from provided token because it is not of type ResponseContinuationToken."); + } + + ReadOnlyMemory data = ((ResponseContinuationToken)token).ToBytes(); + + if (data.Length == 0) + { + Throw.ArgumentException(nameof(token), "Failed to create OpenAIResponsesResumptionToken from provided token because it does not contain any data."); + } + + Utf8JsonReader reader = new(data.Span); + + string? responseId = null; + int? sequenceNumber = null; + + _ = reader.Read(); + + while (reader.Read()) + { + if (reader.TokenType == JsonTokenType.EndObject) + { + break; + } + + string propertyName = reader.GetString()!; + + switch (propertyName) + { + case "responseId": + _ = reader.Read(); + responseId = reader.GetString(); + break; + case "sequenceNumber": + _ = reader.Read(); + sequenceNumber = reader.GetInt32(); + break; + default: + Throw.ArgumentException(nameof(token), $"Unrecognized property '{propertyName}'."); + break; + } + } + + if (responseId is null) + { + Throw.ArgumentException(nameof(token), "Failed to create MessagesPageToken from provided pageToken because it does not contain a responseId."); + } + + return new(responseId) + { + SequenceNumber = sequenceNumber + }; + } +} diff --git a/src/Libraries/Microsoft.Extensions.AI/ChatCompletion/FunctionInvokingChatClient.cs b/src/Libraries/Microsoft.Extensions.AI/ChatCompletion/FunctionInvokingChatClient.cs index ff71e9f83ed..fb821c984df 100644 --- a/src/Libraries/Microsoft.Extensions.AI/ChatCompletion/FunctionInvokingChatClient.cs +++ b/src/Libraries/Microsoft.Extensions.AI/ChatCompletion/FunctionInvokingChatClient.cs @@ -806,6 +806,19 @@ private static void UpdateOptionsForNextIteration(ref ChatOptions? options, stri options = options.Clone(); options.ConversationId = conversationId; } + else if (options.ContinuationToken is not null) + { + // Clone options before resetting the continuation token below. + options = options.Clone(); + } + + // Reset the continuation token of a background response operation + // to signal the inner client to handle function call result rather + // than getting the result of the operation. + if (options?.ContinuationToken is not null) + { + options.ContinuationToken = null; + } } /// Gets whether the function calling loop should exit based on the function call requests. diff --git a/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/ChatCompletion/ChatClientExtensionsTests.cs b/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/ChatCompletion/ChatClientExtensionsTests.cs index c74c50813f4..d5a474f0309 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/ChatCompletion/ChatClientExtensionsTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/ChatCompletion/ChatClientExtensionsTests.cs @@ -158,6 +158,92 @@ public async Task GetStreamingResponseAsync_CreatesTextMessageAsync() Assert.Equal(1, count); } + [Fact] + public async Task GetResponseAsync_UsesProvidedContinuationToken() + { + var expectedResponse = new ChatResponse(); + var expectedContinuationToken = ResponseContinuationToken.FromBytes(new byte[] { 1, 2, 3, 4 }); + var expectedChatOptions = new ChatOptions + { + ContinuationToken = expectedContinuationToken, + AllowBackgroundResponses = true, + AdditionalProperties = new AdditionalPropertiesDictionary // Setting this to ensure cloning is happening + { + { "key", "value" }, + }, + }; + + using var cts = new CancellationTokenSource(); + + using TestChatClient client = new() + { + GetResponseAsyncCallback = (messages, options, cancellationToken) => + { + Assert.Empty(messages); + Assert.NotNull(options); + + Assert.True(options.AdditionalProperties!.ContainsKey("key")); // Assert that chat options were cloned + + Assert.Same(expectedChatOptions, options); + Assert.Same(expectedContinuationToken, options.ContinuationToken); + Assert.Equal(expectedChatOptions.AllowBackgroundResponses, options.AllowBackgroundResponses); + + Assert.Equal(cts.Token, cancellationToken); + + return Task.FromResult(expectedResponse); + }, + }; + + ChatResponse response = await client.GetResponseAsync([], expectedChatOptions, cts.Token); + + Assert.Same(expectedResponse, response); + } + + [Fact] + public async Task GetStreamingResponseAsync_UsesProvidedContinuationToken() + { + var expectedOptions = new ChatOptions(); + var expectedContinuationToken = ResponseContinuationToken.FromBytes(new byte[] { 1, 2, 3, 4 }); + var expectedChatOptions = new ChatOptions + { + ContinuationToken = expectedContinuationToken, + AllowBackgroundResponses = true, + AdditionalProperties = new AdditionalPropertiesDictionary // Setting this to ensure cloning is happening + { + { "key", "value" }, + }, + }; + using var cts = new CancellationTokenSource(); + + using TestChatClient client = new() + { + GetStreamingResponseAsyncCallback = (messages, options, cancellationToken) => + { + Assert.Empty(messages); + Assert.NotNull(options); + + Assert.True(options.AdditionalProperties!.ContainsKey("key")); // Assert that chat options were cloned + + Assert.Same(expectedChatOptions, options); + Assert.Same(expectedContinuationToken, options.ContinuationToken); + Assert.Equal(expectedChatOptions.AllowBackgroundResponses, options.AllowBackgroundResponses); + + Assert.Equal(cts.Token, cancellationToken); + + return YieldAsync([new ChatResponseUpdate(ChatRole.Assistant, "world")]); + }, + }; + + int count = 0; + await foreach (var update in client.GetStreamingResponseAsync([], expectedChatOptions, cts.Token)) + { + Assert.Equal(0, count); + count++; + } + + Assert.Equal(1, count); + } + private static async IAsyncEnumerable YieldAsync(params ChatResponseUpdate[] updates) { await Task.Yield(); diff --git a/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/ChatCompletion/ChatOptionsTests.cs b/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/ChatCompletion/ChatOptionsTests.cs index 5c9fec9111d..e6d863220e1 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/ChatCompletion/ChatOptionsTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/ChatCompletion/ChatOptionsTests.cs @@ -50,6 +50,8 @@ public void Constructor_Parameterless_PropsDefaulted() Assert.Null(clone.Tools); Assert.Null(clone.AdditionalProperties); Assert.Null(clone.RawRepresentationFactory); + Assert.Null(clone.ContinuationToken); + Assert.Null(clone.AllowBackgroundResponses); } [Fact] @@ -76,6 +78,8 @@ public void Properties_Roundtrip() Func rawRepresentationFactory = (c) => null; + ResponseContinuationToken continuationToken = ResponseContinuationToken.FromBytes(new byte[] { 1, 2, 3, 4 }); + options.ConversationId = "12345"; options.Instructions = "Some instructions"; options.Temperature = 0.1f; @@ -93,6 +97,8 @@ public void Properties_Roundtrip() options.Tools = tools; options.RawRepresentationFactory = rawRepresentationFactory; options.AdditionalProperties = additionalProps; + options.ContinuationToken = continuationToken; + options.AllowBackgroundResponses = true; Assert.Equal("12345", options.ConversationId); Assert.Equal("Some instructions", options.Instructions); @@ -111,6 +117,8 @@ public void Properties_Roundtrip() Assert.Same(tools, options.Tools); Assert.Same(rawRepresentationFactory, options.RawRepresentationFactory); Assert.Same(additionalProps, options.AdditionalProperties); + Assert.Same(continuationToken, options.ContinuationToken); + Assert.True(options.AllowBackgroundResponses); ChatOptions clone = options.Clone(); Assert.Equal("12345", clone.ConversationId); @@ -129,6 +137,8 @@ public void Properties_Roundtrip() Assert.Equal(tools, clone.Tools); Assert.Same(rawRepresentationFactory, clone.RawRepresentationFactory); Assert.Equal(additionalProps, clone.AdditionalProperties); + Assert.Same(continuationToken, clone.ContinuationToken); + Assert.True(clone.AllowBackgroundResponses); } [Fact] @@ -147,6 +157,8 @@ public void JsonSerialization_Roundtrips() ["key"] = "value", }; + ResponseContinuationToken continuationToken = ResponseContinuationToken.FromBytes(new byte[] { 1, 2, 3, 4 }); + options.ConversationId = "12345"; options.Instructions = "Some instructions"; options.Temperature = 0.1f; @@ -168,6 +180,8 @@ public void JsonSerialization_Roundtrips() ]; options.RawRepresentationFactory = (c) => null; options.AdditionalProperties = additionalProps; + options.ContinuationToken = continuationToken; + options.AllowBackgroundResponses = true; string json = JsonSerializer.Serialize(options, TestJsonSerializerContext.Default.ChatOptions); diff --git a/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/ResponseContinuationTokenTests.cs b/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/ResponseContinuationTokenTests.cs new file mode 100644 index 00000000000..a6d443a4f47 --- /dev/null +++ b/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/ResponseContinuationTokenTests.cs @@ -0,0 +1,38 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Text.Json; +using Xunit; + +namespace Microsoft.Extensions.AI; + +public class ResponseContinuationTokenTests +{ + [Theory] + [InlineData(new byte[0])] + [InlineData(new byte[] { 1, 2, 3, 4, 5 })] + public void Bytes_Roundtrip(byte[] testBytes) + { + ResponseContinuationToken token = ResponseContinuationToken.FromBytes(testBytes); + + Assert.NotNull(token); + Assert.Equal(testBytes, token.ToBytes().ToArray()); + } + + [Fact] + public void JsonSerialization_Roundtrips() + { + ResponseContinuationToken originalToken = ResponseContinuationToken.FromBytes(new byte[] { 1, 2, 3, 4, 5 }); + + // Act + string json = JsonSerializer.Serialize(originalToken, TestJsonSerializerContext.Default.ResponseContinuationToken); + + ResponseContinuationToken? deserializedToken = JsonSerializer.Deserialize(json, TestJsonSerializerContext.Default.ResponseContinuationToken); + + // Assert + Assert.NotNull(deserializedToken); + Assert.Equal(originalToken.ToBytes().ToArray(), deserializedToken.ToBytes().ToArray()); + Assert.NotSame(originalToken, deserializedToken); + } +} diff --git a/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/TestJsonSerializerContext.cs b/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/TestJsonSerializerContext.cs index 93c7a124e38..d011f8a9030 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/TestJsonSerializerContext.cs +++ b/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/TestJsonSerializerContext.cs @@ -37,4 +37,5 @@ namespace Microsoft.Extensions.AI; [JsonSerializable(typeof(decimal))] // Used in Content tests [JsonSerializable(typeof(HostedMcpServerToolApprovalMode))] [JsonSerializable(typeof(ChatResponseFormatTests.SomeType))] +[JsonSerializable(typeof(ResponseContinuationToken))] internal sealed partial class TestJsonSerializerContext : JsonSerializerContext; diff --git a/test/Libraries/Microsoft.Extensions.AI.Integration.Tests/HttpHandlerExpectedInput.cs b/test/Libraries/Microsoft.Extensions.AI.Integration.Tests/HttpHandlerExpectedInput.cs new file mode 100644 index 00000000000..981e43912c3 --- /dev/null +++ b/test/Libraries/Microsoft.Extensions.AI.Integration.Tests/HttpHandlerExpectedInput.cs @@ -0,0 +1,22 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Net.Http; + +namespace Microsoft.Extensions.AI; + +/// Model for expected input to an HTTP handler. +public sealed class HttpHandlerExpectedInput +{ + /// Gets or sets the expected request URI. + public Uri? Uri { get; set; } + + /// Gets or sets the expected request body. + public string? Body { get; set; } + + /// + /// Gets or sets the expected HTTP method. + /// + public HttpMethod? Method { get; set; } +} diff --git a/test/Libraries/Microsoft.Extensions.AI.Integration.Tests/VerbatimHttpHandler.cs b/test/Libraries/Microsoft.Extensions.AI.Integration.Tests/VerbatimHttpHandler.cs index 8b5f1973348..dd3ea7ea199 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Integration.Tests/VerbatimHttpHandler.cs +++ b/test/Libraries/Microsoft.Extensions.AI.Integration.Tests/VerbatimHttpHandler.cs @@ -21,36 +21,68 @@ namespace Microsoft.Extensions.AI; /// An that checks the request body against an expected one /// and sends back an expected response. /// -public sealed class VerbatimHttpHandler(string expectedInput, string expectedOutput, bool validateExpectedResponse = false) : - DelegatingHandler(new HttpClientHandler()) +public sealed class VerbatimHttpHandler : DelegatingHandler { + private readonly string _expectedOutput; + private readonly bool _validateExpectedResponse; + private readonly HttpHandlerExpectedInput _expectedInput; + + public VerbatimHttpHandler(string expectedInput, string expectedOutput, bool validateExpectedResponse = false) + : this(new HttpHandlerExpectedInput { Body = expectedInput }, expectedOutput, validateExpectedResponse) + { + } + + public VerbatimHttpHandler(HttpHandlerExpectedInput expectedInput, string expectedOutput, bool validateExpectedResponse = false) + : base(new HttpClientHandler()) + { + _expectedOutput = expectedOutput; + _validateExpectedResponse = validateExpectedResponse; + _expectedInput = expectedInput; + } + protected override async Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) { - Assert.NotNull(request.Content); + if (_expectedInput.Body is not null) + { + Assert.NotNull(request.Content); - string? actualInput = await request.Content.ReadAsStringAsync().ConfigureAwait(false); + string? actualInput = await request.Content.ReadAsStringAsync().ConfigureAwait(false); - Assert.NotNull(actualInput); - AssertEqualNormalized(expectedInput, actualInput); + Assert.NotNull(actualInput); + AssertEqualNormalized(_expectedInput.Body, actualInput); - if (validateExpectedResponse) - { - ByteArrayContent newContent = new(Encoding.UTF8.GetBytes(actualInput)); - foreach (var header in request.Content.Headers) + if (_validateExpectedResponse) { - newContent.Headers.TryAddWithoutValidation(header.Key, header.Value); + ByteArrayContent newContent = new(Encoding.UTF8.GetBytes(actualInput)); + foreach (var header in request.Content.Headers) + { + newContent.Headers.TryAddWithoutValidation(header.Key, header.Value); + } + + request.Content = newContent; } + } + + if (_expectedInput.Uri is not null) + { + Assert.Equal(_expectedInput.Uri, request.RequestUri); + } - request.Content = newContent; + if (_expectedInput.Method is not null) + { + Assert.Equal(_expectedInput.Method, request.Method); + } + if (_validateExpectedResponse) + { using var response = await base.SendAsync(request, cancellationToken).ConfigureAwait(false); string? actualOutput = await response.Content.ReadAsStringAsync().ConfigureAwait(false); Assert.NotNull(actualOutput); - AssertEqualNormalized(expectedOutput, actualOutput); + AssertEqualNormalized(_expectedOutput, actualOutput); } - return new() { Content = new StringContent(expectedOutput) }; + return new() { Content = new StringContent(_expectedOutput) }; } public static string? RemoveWhiteSpace(string? text) => diff --git a/test/Libraries/Microsoft.Extensions.AI.OpenAI.Tests/OpenAIResponseClientIntegrationTests.cs b/test/Libraries/Microsoft.Extensions.AI.OpenAI.Tests/OpenAIResponseClientIntegrationTests.cs index c8bdc819ddb..35d72e09436 100644 --- a/test/Libraries/Microsoft.Extensions.AI.OpenAI.Tests/OpenAIResponseClientIntegrationTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.OpenAI.Tests/OpenAIResponseClientIntegrationTests.cs @@ -1,6 +1,7 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. +using System; using System.Collections.Generic; using System.Linq; using System.Threading.Tasks; @@ -189,4 +190,152 @@ await client.GetStreamingResponseAsync(input, chatOptions).ToChatResponseAsync() Assert.Contains("src/Libraries/Microsoft.Extensions.AI.Abstractions/README.md", response.Text); } } + + [ConditionalFact] + public async Task GetResponseAsync_BackgroundResponses() + { + SkipIfNotEnabled(); + + var chatOptions = new ChatOptions + { + AllowBackgroundResponses = true, + }; + + // Get initial response with continuation token + var response = await ChatClient.GetResponseAsync("What's the biggest animal?", chatOptions); + Assert.NotNull(response.ContinuationToken); + Assert.Empty(response.Messages); + + int attempts = 0; + + // Continue to poll until we get the final response + while (response.ContinuationToken is not null && ++attempts < 10) + { + chatOptions.ContinuationToken = response.ContinuationToken; + response = await ChatClient.GetResponseAsync([], chatOptions); + await Task.Delay(1000); + } + + Assert.Contains("whale", response.Text, StringComparison.OrdinalIgnoreCase); + } + + [ConditionalFact] + public async Task GetResponseAsync_BackgroundResponses_WithFunction() + { + SkipIfNotEnabled(); + + int callCount = 0; + + using var chatClient = new FunctionInvokingChatClient(ChatClient); + + var chatOptions = new ChatOptions + { + AllowBackgroundResponses = true, + Tools = [AIFunctionFactory.Create(() => { callCount++; return "5:43"; }, new AIFunctionFactoryOptions { Name = "GetCurrentTime" })] + }; + + // Get initial response with continuation token + var response = await chatClient.GetResponseAsync("What time is it?", chatOptions); + Assert.NotNull(response.ContinuationToken); + Assert.Empty(response.Messages); + + int attempts = 0; + + // Poll until the result is received + while (response.ContinuationToken is not null && ++attempts < 10) + { + chatOptions.ContinuationToken = response.ContinuationToken; + + response = await chatClient.GetResponseAsync([], chatOptions); + await Task.Delay(1000); + } + + Assert.Contains("5:43", response.Text, StringComparison.OrdinalIgnoreCase); + Assert.Equal(1, callCount); + } + + [ConditionalFact] + public async Task GetStreamingResponseAsync_BackgroundResponses() + { + SkipIfNotEnabled(); + + ChatOptions chatOptions = new() + { + AllowBackgroundResponses = true, + }; + + string responseText = ""; + + await foreach (var update in ChatClient.GetStreamingResponseAsync("What is the capital of France?", chatOptions)) + { + responseText += update; + } + + // Assert + Assert.Contains("Paris", responseText, StringComparison.OrdinalIgnoreCase); + } + + [ConditionalFact] + public async Task GetStreamingResponseAsync_BackgroundResponses_StreamResumption() + { + SkipIfNotEnabled(); + + ChatOptions chatOptions = new() + { + AllowBackgroundResponses = true, + }; + + int updateNumber = 0; + string responseText = ""; + object? continuationToken = null; + + await foreach (var update in ChatClient.GetStreamingResponseAsync("What is the capital of France?", chatOptions)) + { + responseText += update; + + // Simulate an interruption after receiving 8 updates. + if (updateNumber++ == 8) + { + continuationToken = update.ContinuationToken; + break; + } + } + + Assert.DoesNotContain("Paris", responseText); + + // Resume streaming from the point of interruption captured by the continuation token. + chatOptions.ContinuationToken = continuationToken; + await foreach (var update in ChatClient.GetStreamingResponseAsync([], chatOptions)) + { + responseText += update; + } + + Assert.Contains("Paris", responseText, StringComparison.OrdinalIgnoreCase); + } + + [ConditionalFact] + public async Task GetStreamingResponseAsync_BackgroundResponses_WithFunction() + { + SkipIfNotEnabled(); + + int callCount = 0; + + using var chatClient = new FunctionInvokingChatClient(ChatClient); + + var chatOptions = new ChatOptions + { + AllowBackgroundResponses = true, + Tools = [AIFunctionFactory.Create(() => { callCount++; return "5:43"; }, new AIFunctionFactoryOptions { Name = "GetCurrentTime" })] + }; + + string responseText = ""; + + await foreach (var update in chatClient.GetStreamingResponseAsync("What time is it?", chatOptions)) + { + responseText += update; + } + + Assert.Contains("5:43", responseText); + Assert.Equal(1, callCount); + } } diff --git a/test/Libraries/Microsoft.Extensions.AI.OpenAI.Tests/OpenAIResponseClientTests.cs b/test/Libraries/Microsoft.Extensions.AI.OpenAI.Tests/OpenAIResponseClientTests.cs index 79e63162923..c4c6f6b767d 100644 --- a/test/Libraries/Microsoft.Extensions.AI.OpenAI.Tests/OpenAIResponseClientTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.OpenAI.Tests/OpenAIResponseClientTests.cs @@ -6,6 +6,7 @@ using System.ClientModel.Primitives; using System.Collections.Generic; using System.ComponentModel; +using System.IO; using System.Linq; using System.Net.Http; using System.Text.Json; @@ -1506,6 +1507,466 @@ public async Task McpToolCall_ApprovalNotRequired_Streaming() Assert.Equal(1569, response.Usage.TotalTokenCount); } + [Fact] + public async Task GetResponseAsync_BackgroundResponses_FirstCall() + { + const string Input = """ + { + "temperature":0.5, + "model":"gpt-4o-mini", + "background":true, + "input": [{ + "type":"message", + "role":"user", + "content":[{"type":"input_text","text":"hello"}] + }], + "max_output_tokens":20 + } + """; + + const string Output = """ + { + "id": "resp_68d3d2c9ef7c8195863e4e2b2ec226a205007262ecbbfed7", + "object": "response", + "created_at": 1758712522, + "status": "queued", + "background": true, + "error": null, + "incomplete_details": null, + "instructions": null, + "max_output_tokens": null, + "model": "gpt-4o-mini-2024-07-18", + "output": [], + "parallel_tool_calls": true, + "previous_response_id": null, + "reasoning": { + "effort": null, + "generate_summary": null + }, + "store": true, + "temperature": 0.5, + "text": { + "format": { + "type": "text" + } + }, + "tool_choice": "auto", + "tools": [], + "top_p": 1.0, + "usage": null, + "user": null, + "metadata": {} + } + """; + + using VerbatimHttpHandler handler = new(Input, Output); + using HttpClient httpClient = new(handler); + using IChatClient client = CreateResponseClient(httpClient, "gpt-4o-mini"); + + var response = await client.GetResponseAsync("hello", new() + { + MaxOutputTokens = 20, + Temperature = 0.5f, + AllowBackgroundResponses = true, + }); + Assert.NotNull(response); + + Assert.Equal("resp_68d3d2c9ef7c8195863e4e2b2ec226a205007262ecbbfed7", response.ResponseId); + Assert.Equal("resp_68d3d2c9ef7c8195863e4e2b2ec226a205007262ecbbfed7", response.ConversationId); + Assert.Empty(response.Messages); + + Assert.NotNull(response.ContinuationToken); + var responsesContinuationToken = TestOpenAIResponsesContinuationToken.FromToken(response.ContinuationToken); + Assert.Equal("resp_68d3d2c9ef7c8195863e4e2b2ec226a205007262ecbbfed7", responsesContinuationToken.ResponseId); + Assert.Null(responsesContinuationToken.SequenceNumber); + } + + [Theory] + [InlineData(ResponseStatus.Queued)] + [InlineData(ResponseStatus.InProgress)] + [InlineData(ResponseStatus.Completed)] + [InlineData(ResponseStatus.Cancelled)] + [InlineData(ResponseStatus.Failed)] + [InlineData(ResponseStatus.Incomplete)] + public async Task GetResponseAsync_BackgroundResponses_PollingCall(ResponseStatus expectedStatus) + { + var expectedInput = new HttpHandlerExpectedInput + { + Uri = new Uri("https://api.openai.com/v1/responses/resp_68d3d2c9ef7c8195863e4e2b2ec226a205007262ecbbfed8"), + Method = HttpMethod.Get, + }; + + string output = $$"""" + { + "id": "resp_68d3d2c9ef7c8195863e4e2b2ec226a205007262ecbbfed8", + "object": "response", + "created_at": 1758712522, + "status": "{{ResponseStatusToRequestValue(expectedStatus)}}", + "background": true, + "error": null, + "incomplete_details": null, + "instructions": null, + "max_output_tokens": null, + "model": "gpt-4o-mini-2024-07-18", + "output": {{(expectedStatus is (ResponseStatus.Queued or ResponseStatus.InProgress) + ? "[]" + : """ + [{ + "type": "message", + "id": "msg_67d32764fcdc8191bcf2e444d4088804058a5e08c46a181d", + "status": "completed", + "role": "assistant", + "content": [ + { + "type": "output_text", + "text": "The background response result.", + "annotations": [] + } + ] + }] + """ + )}}, + "parallel_tool_calls": true, + "previous_response_id": null, + "reasoning": { + "effort": null, + "generate_summary": null + }, + "store": true, + "temperature": 0.5, + "text": { + "format": { + "type": "text" + } + }, + "tool_choice": "auto", + "tools": [], + "top_p": 1.0, + "usage": null, + "user": null, + "metadata": {} + } + """"; + + using VerbatimHttpHandler handler = new(expectedInput, output); + using HttpClient httpClient = new(handler); + using IChatClient client = CreateResponseClient(httpClient, "gpt-4o-mini"); + + var continuationToken = new TestOpenAIResponsesContinuationToken("resp_68d3d2c9ef7c8195863e4e2b2ec226a205007262ecbbfed8"); + + var response = await client.GetResponseAsync([], new() + { + ContinuationToken = continuationToken, + AllowBackgroundResponses = true, + }); + Assert.NotNull(response); + + Assert.Equal("resp_68d3d2c9ef7c8195863e4e2b2ec226a205007262ecbbfed8", response.ResponseId); + Assert.Equal("resp_68d3d2c9ef7c8195863e4e2b2ec226a205007262ecbbfed8", response.ConversationId); + + switch (expectedStatus) + { + case ResponseStatus.Queued: + case ResponseStatus.InProgress: + { + Assert.NotNull(response.ContinuationToken); + + var responsesContinuationToken = TestOpenAIResponsesContinuationToken.FromToken(response.ContinuationToken); + Assert.Equal("resp_68d3d2c9ef7c8195863e4e2b2ec226a205007262ecbbfed8", responsesContinuationToken.ResponseId); + Assert.Null(responsesContinuationToken.SequenceNumber); + + Assert.Empty(response.Messages); + break; + } + + case ResponseStatus.Completed: + case ResponseStatus.Cancelled: + case ResponseStatus.Failed: + case ResponseStatus.Incomplete: + { + Assert.Null(response.ContinuationToken); + + Assert.Equal("The background response result.", response.Text); + Assert.Single(response.Messages.Single().Contents); + Assert.Equal(ChatRole.Assistant, response.Messages.Single().Role); + break; + } + + default: + throw new ArgumentOutOfRangeException(nameof(expectedStatus), expectedStatus, null); + } + } + + [Fact] + public async Task GetResponseAsync_BackgroundResponses_PollingCall_WithMessages() + { + using VerbatimHttpHandler handler = new(string.Empty, string.Empty); + using HttpClient httpClient = new(handler); + using IChatClient client = CreateResponseClient(httpClient, "gpt-4o-mini"); + + var options = new ChatOptions + { + ContinuationToken = new TestOpenAIResponsesContinuationToken("resp_68d3d2c9ef7c8195863e4e2b2ec226a205007262ecbbfed8"), + AllowBackgroundResponses = true, + }; + + // A try to update a background response with new messages should fail. + await Assert.ThrowsAsync(async () => + { + await client.GetResponseAsync("Please book hotel as well", options); + }); + } + + [Fact] + public async Task GetStreamingResponseAsync_BackgroundResponses() + { + const string Input = """ + { + "model": "gpt-4o-2024-08-06", + "background": true, + "input":[{ + "type":"message", + "role":"user", + "content":[{ + "type":"input_text", + "text":"hello" + }] + }], + + "stream": true + } + """; + + const string Output = """ + event: response.created + data: {"type":"response.created","sequence_number":0,"response":{"id":"resp_68d401a7b36c81a288600e95a5a119d4073420ed59d5f559","object":"response","created_at":1758724519,"status":"queued","background":true,"error":null,"incomplete_details":null,"instructions":null,"max_output_tokens":null,"max_tool_calls":null,"model":"gpt-4o-2024-08-06","output":[],"parallel_tool_calls":true,"previous_response_id":null,"prompt_cache_key":null,"reasoning":{"effort":null,"summary":null},"safety_identifier":null,"service_tier":"auto","store":true,"temperature":1.0,"text":{"format":{"type":"text"},"verbosity":"medium"},"tool_choice":"auto","tools":[],"top_logprobs":0,"top_p":1.0,"truncation":"disabled","usage":null,"user":null,"metadata":{}}} + + event: response.queued + data: {"type":"response.queued","sequence_number":1,"response":{"id":"resp_68d401a7b36c81a288600e95a5a119d4073420ed59d5f559","object":"response","created_at":1758724519,"status":"queued","background":true,"error":null,"incomplete_details":null,"instructions":null,"max_output_tokens":null,"max_tool_calls":null,"model":"gpt-4o-2024-08-06","output":[],"parallel_tool_calls":true,"previous_response_id":null,"prompt_cache_key":null,"reasoning":{"effort":null,"summary":null},"safety_identifier":null,"service_tier":"auto","store":true,"temperature":1.0,"text":{"format":{"type":"text"},"verbosity":"medium"},"tool_choice":"auto","tools":[],"top_logprobs":0,"top_p":1.0,"truncation":"disabled","usage":null,"user":null,"metadata":{}}} + + event: response.in_progress + data: {"type":"response.in_progress","sequence_number":2,"response":{"truncation":"disabled","id":"resp_68d401a7b36c81a288600e95a5a119d4073420ed59d5f559","tool_choice":"auto","temperature":1.0,"top_p":1.0,"status":"in_progress","top_logprobs":0,"usage":null,"object":"response","created_at":1758724519,"prompt_cache_key":null,"text":{"format":{"type":"text"},"verbosity":"medium"},"incomplete_details":null,"model":"gpt-4o-2024-08-06","previous_response_id":null,"safety_identifier":null,"metadata":{},"store":true,"output":[],"parallel_tool_calls":true,"error":null,"background":true,"instructions":null,"service_tier":"auto","max_tool_calls":null,"max_output_tokens":null,"tools":[],"user":null,"reasoning":{"effort":null,"summary":null}}} + + event: response.output_item.added + data: {"type":"response.output_item.added","sequence_number":3,"item":{"id":"msg_68d401aa78d481a2ab30776a79c691a6073420ed59d5f559","content":[],"role":"assistant","status":"in_progress","type":"message"},"output_index":0} + + event: response.content_part.added + data: {"type":"response.content_part.added","sequence_number":4,"item_id":"msg_68d401aa78d481a2ab30776a79c691a6073420ed59d5f559","content_index":0,"part":{"text":"","logprobs":[],"type":"output_text","annotations":[]},"output_index":0} + + event: response.output_text.delta + data: {"type":"response.output_text.delta","sequence_number":5,"delta":"Hello","logprobs":[],"item_id":"msg_68d401aa78d481a2ab30776a79c691a6073420ed59d5f559","content_index":0,"output_index":0} + + event: response.output_text.delta + data: {"type":"response.output_text.delta","sequence_number":6,"delta":"!","logprobs":[],"item_id":"msg_68d401aa78d481a2ab30776a79c691a6073420ed59d5f559","content_index":0,"output_index":0} + + event: response.output_text.delta + data: {"type":"response.output_text.delta","sequence_number":7,"delta":" How","logprobs":[],"item_id":"msg_68d401aa78d481a2ab30776a79c691a6073420ed59d5f559","content_index":0,"output_index":0} + + event: response.output_text.delta + data: {"type":"response.output_text.delta","sequence_number":8,"delta":" can","logprobs":[],"item_id":"msg_68d401aa78d481a2ab30776a79c691a6073420ed59d5f559","content_index":0,"output_index":0} + + event: response.output_text.delta + data: {"type":"response.output_text.delta","sequence_number":9,"delta":" I","logprobs":[],"item_id":"msg_68d401aa78d481a2ab30776a79c691a6073420ed59d5f559","content_index":0,"output_index":0} + + event: response.output_text.delta + data: {"type":"response.output_text.delta","sequence_number":10,"delta":" assist","logprobs":[],"item_id":"msg_68d401aa78d481a2ab30776a79c691a6073420ed59d5f559","content_index":0,"output_index":0} + + event: response.output_text.delta + data: {"type":"response.output_text.delta","sequence_number":11,"delta":" you","logprobs":[],"item_id":"msg_68d401aa78d481a2ab30776a79c691a6073420ed59d5f559","content_index":0,"output_index":0} + + event: response.output_text.delta + data: {"type":"response.output_text.delta","sequence_number":12,"delta":" today","logprobs":[],"item_id":"msg_68d401aa78d481a2ab30776a79c691a6073420ed59d5f559","content_index":0,"output_index":0} + + event: response.output_text.delta + data: {"type":"response.output_text.delta","sequence_number":13,"delta":"?","logprobs":[],"item_id":"msg_68d401aa78d481a2ab30776a79c691a6073420ed59d5f559","content_index":0,"output_index":0} + + event: response.output_text.done + data: {"type":"response.output_text.done","sequence_number":14,"text":"Hello! How can I assist you today?","logprobs":[],"item_id":"msg_68d401aa78d481a2ab30776a79c691a6073420ed59d5f559","content_index":0,"output_index":0} + + event: response.content_part.done + data: {"type":"response.content_part.done","sequence_number":15,"item_id":"msg_68d401aa78d481a2ab30776a79c691a6073420ed59d5f559","content_index":0,"part":{"text":"Hello! How can I assist you today?","logprobs":[],"type":"output_text","annotations":[]},"output_index":0} + + event: response.output_item.done + data: {"type":"response.output_item.done","sequence_number":16,"item":{"id":"msg_68d401aa78d481a2ab30776a79c691a6073420ed59d5f559","content":[{"text":"Hello! How can I assist you today?","logprobs":[],"type":"output_text","annotations":[]}],"role":"assistant","status":"completed","type":"message"},"output_index":0} + + event: response.completed + data: {"type":"response.completed","sequence_number":17,"response":{"truncation":"disabled","id":"resp_68d401a7b36c81a288600e95a5a119d4073420ed59d5f559","tool_choice":"auto","temperature":1.0,"top_p":1.0,"status":"completed","top_logprobs":0,"usage":{"total_tokens":18,"input_tokens_details":{"cached_tokens":0},"output_tokens_details":{"reasoning_tokens":0},"output_tokens":10,"input_tokens":8},"object":"response","created_at":1758724519,"prompt_cache_key":null,"text":{"format":{"type":"text"},"verbosity":"medium"},"incomplete_details":null,"model":"gpt-4o-2024-08-06","previous_response_id":null,"safety_identifier":null,"metadata":{},"store":true,"output":[{"id":"msg_68d401aa78d481a2ab30776a79c691a6073420ed59d5f559","content":[{"text":"Hello! How can I assist you today?","logprobs":[],"type":"output_text","annotations":[]}],"role":"assistant","status":"completed","type":"message"}],"parallel_tool_calls":true,"error":null,"background":true,"instructions":null,"service_tier":"default","max_tool_calls":null,"max_output_tokens":null,"tools":[],"user":null,"reasoning":{"effort":null,"summary":null}}} + + + """; + + using VerbatimHttpHandler handler = new(Input, Output); + using HttpClient httpClient = new(handler); + using IChatClient client = CreateResponseClient(httpClient, "gpt-4o-2024-08-06"); + + List updates = []; + await foreach (var update in client.GetStreamingResponseAsync("hello", new() + { + AllowBackgroundResponses = true, + })) + { + updates.Add(update); + } + + Assert.Equal("Hello! How can I assist you today?", string.Concat(updates.Select(u => u.Text))); + Assert.Equal(18, updates.Count); + + var createdAt = DateTimeOffset.FromUnixTimeSeconds(1_758_724_519); + + for (int i = 0; i < updates.Count; i++) + { + Assert.Equal("resp_68d401a7b36c81a288600e95a5a119d4073420ed59d5f559", updates[i].ResponseId); + Assert.Equal("resp_68d401a7b36c81a288600e95a5a119d4073420ed59d5f559", updates[i].ConversationId); + Assert.Equal(createdAt, updates[i].CreatedAt); + Assert.Equal("gpt-4o-2024-08-06", updates[i].ModelId); + Assert.Null(updates[i].AdditionalProperties); + + if (i < updates.Count - 1) + { + Assert.NotNull(updates[i].ContinuationToken); + var responsesContinuationToken = TestOpenAIResponsesContinuationToken.FromToken(updates[i].ContinuationToken!); + Assert.Equal("resp_68d401a7b36c81a288600e95a5a119d4073420ed59d5f559", responsesContinuationToken.ResponseId); + Assert.Equal(i, responsesContinuationToken.SequenceNumber); + Assert.Null(updates[i].FinishReason); + } + else + { + Assert.Null(updates[i].ContinuationToken); + Assert.Equal(ChatFinishReason.Stop, updates[i].FinishReason); + } + + Assert.Equal((i >= 5 && i <= 13) || i == 17 ? 1 : 0, updates[i].Contents.Count); + } + } + + [Fact] + public async Task GetStreamingResponseAsync_BackgroundResponses_StreamResumption() + { + var expectedInput = new HttpHandlerExpectedInput + { + Uri = new Uri("https://api.openai.com/v1/responses/resp_68d40dc671a0819cb0ee920078333451029e611c3cc4a34b?stream=true&starting_after=9"), + Method = HttpMethod.Get, + }; + + const string Output = """ + event: response.output_text.delta + data: {"type":"response.output_text.delta","sequence_number":10,"delta":" assist","logprobs":[],"item_id":"msg_68d40dcb2d34819c88f5d6a8ca7b0308029e611c3cc4a34b","content_index":0,"output_index":0} + + event: response.output_text.delta + data: {"type":"response.output_text.delta","sequence_number":11,"delta":" you","logprobs":[],"item_id":"msg_68d40dcb2d34819c88f5d6a8ca7b0308029e611c3cc4a34b","content_index":0,"output_index":0} + + event: response.output_text.delta + data: {"type":"response.output_text.delta","sequence_number":12,"delta":" today","logprobs":[],"item_id":"msg_68d40dcb2d34819c88f5d6a8ca7b0308029e611c3cc4a34b","content_index":0,"output_index":0} + + event: response.output_text.delta + data: {"type":"response.output_text.delta","sequence_number":13,"delta":"?","logprobs":[],"item_id":"msg_68d40dcb2d34819c88f5d6a8ca7b0308029e611c3cc4a34b","content_index":0,"output_index":0} + + event: response.output_text.done + data: {"type":"response.output_text.done","sequence_number":14,"text":"Hello! How can I assist you today?","logprobs":[],"item_id":"msg_68d40dcb2d34819c88f5d6a8ca7b0308029e611c3cc4a34b","content_index":0,"output_index":0} + + event: response.content_part.done + data: {"type":"response.content_part.done","sequence_number":15,"item_id":"msg_68d40dcb2d34819c88f5d6a8ca7b0308029e611c3cc4a34b","content_index":0,"part":{"text":"Hello! How can I assist you today?","logprobs":[],"type":"output_text","annotations":[]},"output_index":0} + + event: response.output_item.done + data: {"type":"response.output_item.done","sequence_number":16,"item":{"id":"msg_68d40dcb2d34819c88f5d6a8ca7b0308029e611c3cc4a34b","content":[{"text":"Hello! How can I assist you today?","logprobs":[],"type":"output_text","annotations":[]}],"role":"assistant","status":"completed","type":"message"},"output_index":0} + + event: response.completed + data: {"type":"response.completed","sequence_number":17,"response":{"truncation":"disabled","id":"resp_68d40dc671a0819cb0ee920078333451029e611c3cc4a34b","tool_choice":"auto","temperature":1.0,"top_p":1.0,"status":"completed","top_logprobs":0,"usage":{"total_tokens":18,"input_tokens_details":{"cached_tokens":0},"output_tokens_details":{"reasoning_tokens":0},"output_tokens":10,"input_tokens":8},"object":"response","created_at":1758727622,"prompt_cache_key":null,"text":{"format":{"type":"text"},"verbosity":"medium"},"incomplete_details":null,"model":"gpt-4o-2024-08-06","previous_response_id":null,"safety_identifier":null,"metadata":{},"store":true,"output":[{"id":"msg_68d40dcb2d34819c88f5d6a8ca7b0308029e611c3cc4a34b","content":[{"text":"Hello! How can I assist you today?","logprobs":[],"type":"output_text","annotations":[]}],"role":"assistant","status":"completed","type":"message"}],"parallel_tool_calls":true,"error":null,"background":true,"instructions":null,"service_tier":"default","max_tool_calls":null,"max_output_tokens":null,"tools":[],"user":null,"reasoning":{"effort":null,"summary":null}}} + + + """; + + using VerbatimHttpHandler handler = new(expectedInput, Output); + using HttpClient httpClient = new(handler); + using IChatClient client = CreateResponseClient(httpClient, "gpt-4o-2024-08-06"); + + // Emulating resumption of the stream after receiving the first 9 updates that provided the text "Hello! How can I" + var continuationToken = new TestOpenAIResponsesContinuationToken("resp_68d40dc671a0819cb0ee920078333451029e611c3cc4a34b") + { + SequenceNumber = 9 + }; + + var chatOptions = new ChatOptions + { + AllowBackgroundResponses = true, + ContinuationToken = continuationToken, + }; + + List updates = []; + await foreach (var update in client.GetStreamingResponseAsync([], chatOptions)) + { + updates.Add(update); + } + + // Receiving the remaining updates to complete the response "Hello! How can I assist you today?" + Assert.Equal(" assist you today?", string.Concat(updates.Select(u => u.Text))); + Assert.Equal(8, updates.Count); + + var createdAt = DateTimeOffset.FromUnixTimeSeconds(1_758_727_622); + + for (int i = 0; i < updates.Count; i++) + { + Assert.Equal("resp_68d40dc671a0819cb0ee920078333451029e611c3cc4a34b", updates[i].ResponseId); + Assert.Equal("resp_68d40dc671a0819cb0ee920078333451029e611c3cc4a34b", updates[i].ConversationId); + + var sequenceNumber = i + 10; + + if (sequenceNumber is (>= 10 and <= 13)) + { + // Text deltas + Assert.NotNull(updates[i].ContinuationToken); + var responsesContinuationToken = TestOpenAIResponsesContinuationToken.FromToken(updates[i].ContinuationToken!); + Assert.Equal("resp_68d40dc671a0819cb0ee920078333451029e611c3cc4a34b", responsesContinuationToken.ResponseId); + Assert.Equal(sequenceNumber, responsesContinuationToken.SequenceNumber); + + Assert.Single(updates[i].Contents); + } + else if (sequenceNumber is (>= 14 and <= 16)) + { + // Response Complete and Assistant message updates + Assert.NotNull(updates[i].ContinuationToken); + var responsesContinuationToken = TestOpenAIResponsesContinuationToken.FromToken(updates[i].ContinuationToken!); + Assert.Equal("resp_68d40dc671a0819cb0ee920078333451029e611c3cc4a34b", responsesContinuationToken.ResponseId); + Assert.Equal(sequenceNumber, responsesContinuationToken.SequenceNumber); + + Assert.Empty(updates[i].Contents); + } + else + { + // The last update with the response completion + Assert.Null(updates[i].ContinuationToken); + Assert.Single(updates[i].Contents); + } + } + } + + [Fact] + public async Task GetStreamingResponseAsync_BackgroundResponses_StreamResumption_WithMessages() + { + using VerbatimHttpHandler handler = new(string.Empty, string.Empty); + using HttpClient httpClient = new(handler); + using IChatClient client = CreateResponseClient(httpClient, "gpt-4o-2024-08-06"); + + // Emulating resumption of the stream after receiving the first 9 updates that provided the text "Hello! How can I" + var chatOptions = new ChatOptions + { + AllowBackgroundResponses = true, + ContinuationToken = new TestOpenAIResponsesContinuationToken("resp_68d40dc671a0819cb0ee920078333451029e611c3cc4a34b") + { + SequenceNumber = 9 + } + }; + + await Assert.ThrowsAsync(async () => + { + await foreach (var update in client.GetStreamingResponseAsync("Please book a hotel for me", chatOptions)) +#pragma warning disable S108 // Nested blocks of code should not be left empty + { + } +#pragma warning restore S108 // Nested blocks of code should not be left empty + }); + } + [Fact] public async Task RequestHeaders_UserAgent_ContainsMEAI() { @@ -1525,4 +1986,101 @@ private static IChatClient CreateResponseClient(HttpClient httpClient, string mo new OpenAIClientOptions { Transport = new HttpClientPipelineTransport(httpClient) }) .GetOpenAIResponseClient(modelId) .AsIChatClient(); + + private static string ResponseStatusToRequestValue(ResponseStatus status) + { + if (status == ResponseStatus.InProgress) + { + return "in_progress"; + } + + return status.ToString().ToLowerInvariant(); + } + + private sealed class TestOpenAIResponsesContinuationToken : ResponseContinuationToken + { + internal TestOpenAIResponsesContinuationToken(string responseId) + { + ResponseId = responseId; + } + + /// Gets or sets the Id of the response. + internal string ResponseId { get; set; } + + /// Gets or sets the sequence number of a streamed update. + internal int? SequenceNumber { get; set; } + + internal static TestOpenAIResponsesContinuationToken FromToken(object token) + { + if (token is TestOpenAIResponsesContinuationToken testOpenAIResponsesContinuationToken) + { + return testOpenAIResponsesContinuationToken; + } + + if (token is not ResponseContinuationToken) + { + throw new ArgumentException("Failed to create OpenAIResponsesResumptionToken from provided token because it is not of type ResponseContinuationToken.", nameof(token)); + } + + ReadOnlyMemory data = ((ResponseContinuationToken)token).ToBytes(); + + Utf8JsonReader reader = new(data.Span); + + string responseId = null!; + int? startAfter = null; + + _ = reader.Read(); + + while (reader.Read()) + { + if (reader.TokenType == JsonTokenType.EndObject) + { + break; + } + + string propertyName = reader.GetString()!; + + switch (propertyName) + { + case "responseId": + _ = reader.Read(); + responseId = reader.GetString()!; + break; + case "sequenceNumber": + _ = reader.Read(); + startAfter = reader.GetInt32(); + break; + default: + throw new JsonException($"Unrecognized property '{propertyName}'."); + } + } + + return new(responseId) + { + SequenceNumber = startAfter + }; + } + + public override ReadOnlyMemory ToBytes() + { + using MemoryStream stream = new(); + using Utf8JsonWriter writer = new(stream); + + writer.WriteStartObject(); + + writer.WriteString("responseId", ResponseId); + + if (SequenceNumber.HasValue) + { + writer.WriteNumber("sequenceNumber", SequenceNumber.Value); + } + + writer.WriteEndObject(); + + writer.Flush(); + stream.Position = 0; + + return stream.ToArray(); + } + } } diff --git a/test/Libraries/Microsoft.Extensions.AI.Tests/ChatCompletion/FunctionInvokingChatClientTests.cs b/test/Libraries/Microsoft.Extensions.AI.Tests/ChatCompletion/FunctionInvokingChatClientTests.cs index 27c0d3ff0d6..2308a921ab3 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Tests/ChatCompletion/FunctionInvokingChatClientTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.Tests/ChatCompletion/FunctionInvokingChatClientTests.cs @@ -1194,6 +1194,44 @@ public async Task MixedKnownFunctionAndDeclaration_TerminatesWithoutInvokingKnow Assert.Equal(0, invoked); } + [Fact] + public async Task ClonesChatOptionsAndResetContinuationTokenForBackgroundResponsesAsync() + { + ChatOptions? actualChatOptions = null; + + using var innerChatClient = new TestChatClient + { + GetResponseAsyncCallback = (chatContents, chatOptions, cancellationToken) => + { + actualChatOptions = chatOptions; + + List messages = []; + + // Simulate the model returning a function call for the first call only + if (!chatContents.Any(m => m.Contents.OfType().Any())) + { + messages.Add(new ChatMessage(ChatRole.Assistant, [new FunctionCallContent("callId1", "Func1")])); + } + + return Task.FromResult(new ChatResponse { Messages = messages }); + } + }; + + using var chatClient = new FunctionInvokingChatClient(innerChatClient); + + var originalChatOptions = new ChatOptions + { + Tools = [AIFunctionFactory.Create(() => { }, "Func1")], + ContinuationToken = ResponseContinuationToken.FromBytes(new byte[] { 1, 2, 3, 4 }), + }; + + await chatClient.GetResponseAsync("hi", originalChatOptions); + + // The original options should be cloned and have a null ContinuationToken + Assert.NotSame(originalChatOptions, actualChatOptions); + Assert.Null(actualChatOptions!.ContinuationToken); + } + private sealed class CustomSynchronizationContext : SynchronizationContext { public override void Post(SendOrPostCallback d, object? state)