Add Request/Response History to all public Response types (#817)

Work to close
#790

The fact that `HTTPClient.Request` is not Sendable make me think we're
going to need to store something else, such as a `URL` and
`HTTPRequestHead`, instead?

Author: gregcotten

Sources/AsyncHTTPClient/AsyncAwait/HTTPClient+execute.swift

@@ -85,11 +85,29 @@ extension HTTPClient {
     ) async throws -> HTTPClientResponse {
         var currentRequest = request
         var currentRedirectState = redirectState
+        var history: [HTTPClientRequestResponse] = []
 
         // this loop is there to follow potential redirects
         while true {
             let preparedRequest = try HTTPClientRequest.Prepared(currentRequest, dnsOverride: configuration.dnsOverride)
-            let response = try await self.executeCancellable(preparedRequest, deadline: deadline, logger: logger)
+            let response = try await {
+                var response = try await self.executeCancellable(preparedRequest, deadline: deadline, logger: logger)
+
+                history.append(
+                    .init(
+                        request: currentRequest,
+                        responseHead: .init(
+                            version: response.version,
+                            status: response.status,
+                            headers: response.headers
+                        )
+                    )
+                )
+
+                response.history = history
+
+                return response
+            }()
 
             guard var redirectState = currentRedirectState else {
                 // a `nil` redirectState means we should not follow redirects

Sources/AsyncHTTPClient/AsyncAwait/HTTPClientResponse.swift

@@ -15,6 +15,8 @@
 import NIOCore
 import NIOHTTP1
 
+import struct Foundation.URL
+
 /// A representation of an HTTP response for the Swift Concurrency HTTPClient API.
 ///
 /// This object is similar to ``HTTPClient/Response``, but used for the Swift Concurrency API.
@@ -32,6 +34,18 @@ public struct HTTPClientResponse: Sendable {
     /// The body of this HTTP response.
     public var body: Body
 
+    /// The history of all requests and responses in redirect order.
+    public var history: [HTTPClientRequestResponse]
+
+    /// The target URL (after redirects) of the response.
+    public var url: URL? {
+        guard let lastRequestURL = self.history.last?.request.url else {
+            return nil
+        }
+
+        return URL(string: lastRequestURL)
+    }
+
     @inlinable public init(
         version: HTTPVersion = .http1_1,
         status: HTTPResponseStatus = .ok,
@@ -42,14 +56,30 @@ public struct HTTPClientResponse: Sendable {
         self.status = status
         self.headers = headers
         self.body = body
+        self.history = []
+    }
+
+    @inlinable public init(
+        version: HTTPVersion = .http1_1,
+        status: HTTPResponseStatus = .ok,
+        headers: HTTPHeaders = [:],
+        body: Body = Body(),
+        history: [HTTPClientRequestResponse] = []
+    ) {
+        self.version = version
+        self.status = status
+        self.headers = headers
+        self.body = body
+        self.history = history
     }
 
     init(
         requestMethod: HTTPMethod,
         version: HTTPVersion,
         status: HTTPResponseStatus,
         headers: HTTPHeaders,
-        body: TransactionBody
+        body: TransactionBody,
+        history: [HTTPClientRequestResponse]
     ) {
         self.init(
             version: version,
@@ -64,11 +94,23 @@ public struct HTTPClientResponse: Sendable {
                         status: status
                     )
                 )
-            )
+            ),
+            history: history
         )
     }
 }
 
+@available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *)
+public struct HTTPClientRequestResponse: Sendable {
+    public var request: HTTPClientRequest
+    public var responseHead: HTTPResponseHead
+
+    public init(request: HTTPClientRequest, responseHead: HTTPResponseHead) {
+        self.request = request
+        self.responseHead = responseHead
+    }
+}
+
 @available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *)
 extension HTTPClientResponse {
     /// A representation of the response body for an HTTP response.

Sources/AsyncHTTPClient/AsyncAwait/Transaction.swift

@@ -242,7 +242,8 @@ extension Transaction: HTTPExecutableRequest {
                 version: head.version,
                 status: head.status,
                 headers: head.headers,
-                body: body
+                body: body,
+                history: []
             )
             continuation.resume(returning: response)
         }

Sources/AsyncHTTPClient/FileDownloadDelegate.swift

@@ -16,15 +16,25 @@ import NIOCore
 import NIOHTTP1
 import NIOPosix
 
+import struct Foundation.URL
+
 /// Handles a streaming download to a given file path, allowing headers and progress to be reported.
 public final class FileDownloadDelegate: HTTPClientResponseDelegate {
     /// The response type for this delegate: the total count of bytes as reported by the response
-    /// "Content-Length" header (if available), the count of bytes downloaded, and the
-    /// response head.
+    /// "Content-Length" header (if available), the count of bytes downloaded, the
+    /// response head, and a history of requests and responses.
     public struct Progress: Sendable {
         public var totalBytes: Int?
         public var receivedBytes: Int
 
+        /// The history of all requests and responses in redirect order.
+        public var history: [HTTPClient.RequestResponse] = []
+
+        /// The target URL (after redirects) of the response.
+        public var url: URL? {
+            self.history.last?.request.url
+        }
+
         public var head: HTTPResponseHead {
             get {
                 assert(self._head != nil)
@@ -150,6 +160,10 @@ public final class FileDownloadDelegate: HTTPClientResponseDelegate {
         )
     }
 
+    public func didVisitURL(task: HTTPClient.Task<Progress>, _ request: HTTPClient.Request, _ head: HTTPResponseHead) {
+        self.progress.history.append(.init(request: request, responseHead: head))
+    }
+
     public func didReceiveHead(
         task: HTTPClient.Task<Response>,
         _ head: HTTPResponseHead

Sources/AsyncHTTPClient/HTTPHandler.swift

@@ -13,17 +13,22 @@
 //===----------------------------------------------------------------------===//
 
 import Algorithms
-import Foundation
 import Logging
 import NIOConcurrencyHelpers
 import NIOCore
 import NIOHTTP1
 import NIOPosix
 import NIOSSL
 
+#if compiler(>=6.0)
+import Foundation
+#else
+@preconcurrency import Foundation
+#endif
+
 extension HTTPClient {
     /// A request body.
-    public struct Body {
+    public struct Body: Sendable {
         /// A streaming uploader.
         ///
         /// ``StreamWriter`` abstracts
@@ -209,7 +214,7 @@ extension HTTPClient {
     }
 
     /// Represents an HTTP request.
-    public struct Request {
+    public struct Request: Sendable {
         /// Request HTTP method, defaults to `GET`.
         public let method: HTTPMethod
         /// Remote URL.
@@ -377,6 +382,13 @@ extension HTTPClient {
         public var headers: HTTPHeaders
         /// Response body.
         public var body: ByteBuffer?
+        /// The history of all requests and responses in redirect order.
+        public var history: [RequestResponse]
+
+        /// The target URL (after redirects) of the response.
+        public var url: URL? {
+            self.history.last?.request.url
+        }
 
         /// Create HTTP `Response`.
         ///
@@ -392,6 +404,7 @@ extension HTTPClient {
             self.version = HTTPVersion(major: 1, minor: 1)
             self.headers = headers
             self.body = body
+            self.history = []
         }
 
         /// Create HTTP `Response`.
@@ -414,6 +427,32 @@ extension HTTPClient {
             self.version = version
             self.headers = headers
             self.body = body
+            self.history = []
+        }
+
+        /// Create HTTP `Response`.
+        ///
+        /// - parameters:
+        ///     - host: Remote host of the request.
+        ///     - status: Response HTTP status.
+        ///     - version: Response HTTP version.
+        ///     - headers: Reponse HTTP headers.
+        ///     - body: Response body.
+        ///     - history: History of all requests and responses in redirect order.
+        public init(
+            host: String,
+            status: HTTPResponseStatus,
+            version: HTTPVersion,
+            headers: HTTPHeaders,
+            body: ByteBuffer?,
+            history: [RequestResponse]
+        ) {
+            self.host = host
+            self.status = status
+            self.version = version
+            self.headers = headers
+            self.body = body
+            self.history = history
         }
     }
 
@@ -457,6 +496,16 @@ extension HTTPClient {
             }
         }
     }
+
+    public struct RequestResponse: Sendable {
+        public var request: Request
+        public var responseHead: HTTPResponseHead
+
+        public init(request: Request, responseHead: HTTPResponseHead) {
+            self.request = request
+            self.responseHead = responseHead
+        }
+    }
 }
 
 /// The default ``HTTPClientResponseDelegate``.
@@ -485,6 +534,7 @@ public final class ResponseAccumulator: HTTPClientResponseDelegate {
         }
     }
 
+    var history = [HTTPClient.RequestResponse]()
     var state = State.idle
     let requestMethod: HTTPMethod
     let requestHost: String
@@ -521,6 +571,14 @@ public final class ResponseAccumulator: HTTPClientResponseDelegate {
         self.maxBodySize = maxBodySize
     }
 
+    public func didVisitURL(
+        task: HTTPClient.Task<HTTPClient.Response>,
+        _ request: HTTPClient.Request,
+        _ head: HTTPResponseHead
+    ) {
+        self.history.append(.init(request: request, responseHead: head))
+    }
+
     public func didReceiveHead(task: HTTPClient.Task<Response>, _ head: HTTPResponseHead) -> EventLoopFuture<Void> {
         switch self.state {
         case .idle:
@@ -596,15 +654,17 @@ public final class ResponseAccumulator: HTTPClientResponseDelegate {
                 status: head.status,
                 version: head.version,
                 headers: head.headers,
-                body: nil
+                body: nil,
+                history: self.history
             )
         case .body(let head, let body):
             return Response(
                 host: self.requestHost,
                 status: head.status,
                 version: head.version,
                 headers: head.headers,
-                body: body
+                body: body,
+                history: self.history
             )
         case .end:
             preconditionFailure("request already processed")

Tests/AsyncHTTPClientTests/AsyncAwaitEndToEndTests.swift

@@ -76,6 +76,8 @@ final class AsyncAwaitEndToEndTests: XCTestCase {
                 return
             }
 
+            XCTAssertEqual(response.url?.absoluteString, request.url)
+            XCTAssertEqual(response.history.map(\.request.url), [request.url])
             XCTAssertEqual(response.status, .ok)
             XCTAssertEqual(response.version, .http2)
         }
@@ -98,6 +100,8 @@ final class AsyncAwaitEndToEndTests: XCTestCase {
                 return
             }
 
+            XCTAssertEqual(response.url?.absoluteString, request.url)
+            XCTAssertEqual(response.history.map(\.request.url), [request.url])
             XCTAssertEqual(response.status, .ok)
             XCTAssertEqual(response.version, .http2)
         }
@@ -734,9 +738,10 @@ final class AsyncAwaitEndToEndTests: XCTestCase {
             defer { XCTAssertNoThrow(try client.syncShutdown()) }
             let logger = Logger(label: "HTTPClient", factory: StreamLogHandler.standardOutput(label:))
             var request = HTTPClientRequest(url: "https://127.0.0.1:\(bin.port)/redirect/target")
+            let redirectURL = "https://localhost:\(bin.port)/echohostheader"
             request.headers.replaceOrAdd(
                 name: "X-Target-Redirect-URL",
-                value: "https://localhost:\(bin.port)/echohostheader"
+                value: redirectURL
             )
 
             guard
@@ -753,6 +758,8 @@ final class AsyncAwaitEndToEndTests: XCTestCase {
             XCTAssertNoThrow(maybeRequestInfo = try JSONDecoder().decode(RequestInfo.self, from: body))
             guard let requestInfo = maybeRequestInfo else { return }
 
+            XCTAssertEqual(response.url?.absoluteString, redirectURL)
+            XCTAssertEqual(response.history.map(\.request.url), [request.url, redirectURL])
             XCTAssertEqual(response.status, .ok)
             XCTAssertEqual(response.version, .http2)
             XCTAssertEqual(requestInfo.data, "localhost:\(bin.port)")