From eb7d0e1878f259a446db922b201b79e98e8c753e Mon Sep 17 00:00:00 2001 From: Dongie Agnir Date: Fri, 19 Sep 2025 11:40:00 -0700 Subject: [PATCH 1/2] Reuse computed checkums for async payloads This is a continuation of #6413 that adds support for checksum reuse on async payloads. Additionally, this commit renames references to "ChecksumCache" to "ChecksumStore". --- .../signer/AwsChunkedV4aPayloadSigner.java | 10 +- .../signer/DefaultAwsCrtV4aHttpSigner.java | 10 +- .../signer/AwsChunkedV4PayloadSigner.java | 10 +- .../signer/DefaultAwsV4HttpSigner.java | 10 +- .../ChecksumTrailerProvider.java | 10 +- .../DefaultAwsCrtV4aHttpSignerTest.java | 14 +-- .../signer/DefaultAwsV4HttpSignerTest.java | 14 +-- .../signer/SdkInternalHttpSignerProperty.java | 6 +- .../SdkInternalExecutionAttribute.java | 6 +- .../ChecksumCalculatingAsyncRequestBody.java | 35 +++++- .../pipeline/stages/HttpChecksumStage.java | 14 ++- .../http/pipeline/stages/SigningStage.java | 4 +- ...ecksumCalculatingAsyncRequestBodyTest.java | 57 ++++++++++ .../stages/HttpChecksumStageSraTest.java | 12 +- .../pipeline/stages/SigningStageTest.java | 8 +- .../s3/checksums/ChecksumReuseTest.java | 104 +++++++++++++++++- 16 files changed, 256 insertions(+), 68 deletions(-) diff --git a/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/crt/internal/signer/AwsChunkedV4aPayloadSigner.java b/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/crt/internal/signer/AwsChunkedV4aPayloadSigner.java index 644e204f34b0..de5d16b92799 100644 --- a/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/crt/internal/signer/AwsChunkedV4aPayloadSigner.java +++ b/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/crt/internal/signer/AwsChunkedV4aPayloadSigner.java @@ -66,8 +66,8 @@ private AwsChunkedV4aPayloadSigner(Builder builder) { this.credentialScope = Validate.paramNotNull(builder.credentialScope, "CredentialScope"); this.chunkSize = Validate.isPositive(builder.chunkSize, "ChunkSize"); this.checksumAlgorithm = builder.checksumAlgorithm; - this.payloadChecksumStore = builder.payloadChecksumStore == null ? NoOpPayloadChecksumStore.create() : - builder.payloadChecksumStore; + this.payloadChecksumStore = builder.checksumStore == null ? NoOpPayloadChecksumStore.create() : + builder.checksumStore; } public static Builder builder() { @@ -282,7 +282,7 @@ static final class Builder { private CredentialScope credentialScope; private Integer chunkSize; private ChecksumAlgorithm checksumAlgorithm; - private PayloadChecksumStore payloadChecksumStore; + private PayloadChecksumStore checksumStore; public Builder credentialScope(CredentialScope credentialScope) { this.credentialScope = credentialScope; @@ -299,8 +299,8 @@ public Builder checksumAlgorithm(ChecksumAlgorithm checksumAlgorithm) { return this; } - public Builder checksumCache(PayloadChecksumStore checksumCache) { - this.payloadChecksumStore = checksumCache; + public Builder checksumStore(PayloadChecksumStore checksumStore) { + this.checksumStore = checksumStore; return this; } diff --git a/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/crt/internal/signer/DefaultAwsCrtV4aHttpSigner.java b/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/crt/internal/signer/DefaultAwsCrtV4aHttpSigner.java index a77385b5c036..af2a14e93823 100644 --- a/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/crt/internal/signer/DefaultAwsCrtV4aHttpSigner.java +++ b/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/crt/internal/signer/DefaultAwsCrtV4aHttpSigner.java @@ -33,7 +33,7 @@ import static software.amazon.awssdk.http.auth.aws.internal.signer.util.CredentialUtils.sanitizeCredentials; import static software.amazon.awssdk.http.auth.aws.internal.signer.util.SignerConstant.PRESIGN_URL_MAX_EXPIRATION_DURATION; import static software.amazon.awssdk.http.auth.aws.internal.signer.util.SignerConstant.X_AMZ_TRAILER; -import static software.amazon.awssdk.http.auth.spi.signer.SdkInternalHttpSignerProperty.CHECKSUM_CACHE; +import static software.amazon.awssdk.http.auth.spi.signer.SdkInternalHttpSignerProperty.CHECKSUM_STORE; import java.time.Clock; import java.time.Duration; @@ -73,7 +73,7 @@ public final class DefaultAwsCrtV4aHttpSigner implements AwsV4aHttpSigner { @Override public SignedRequest sign(SignRequest request) { - Checksummer checksummer = checksummer(request, null, checksumCache(request)); + Checksummer checksummer = checksummer(request, null, checksumStore(request)); V4aProperties v4aProperties = v4aProperties(request); AwsSigningConfig signingConfig = signingConfig(request, v4aProperties); V4aPayloadSigner payloadSigner = v4aPayloadSigner(request, v4aProperties); @@ -120,7 +120,7 @@ private static V4aPayloadSigner v4aPayloadSigner( .credentialScope(v4aProperties.getCredentialScope()) .chunkSize(DEFAULT_CHUNK_SIZE_IN_BYTES) .checksumAlgorithm(request.property(CHECKSUM_ALGORITHM)) - .checksumCache(checksumCache(request)) + .checksumStore(checksumStore(request)) .build(); } @@ -257,8 +257,8 @@ private static V4aRequestSigningResult sign(SdkHttpRequest request, HttpRequest signingConfig); } - private static PayloadChecksumStore checksumCache(SignRequest request) { - PayloadChecksumStore cache = request.property(CHECKSUM_CACHE); + private static PayloadChecksumStore checksumStore(SignRequest request) { + PayloadChecksumStore cache = request.property(CHECKSUM_STORE); if (cache == null) { return NoOpPayloadChecksumStore.create(); } diff --git a/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/signer/AwsChunkedV4PayloadSigner.java b/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/signer/AwsChunkedV4PayloadSigner.java index 4f43701f9bf5..8e2e3a3a168b 100644 --- a/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/signer/AwsChunkedV4PayloadSigner.java +++ b/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/signer/AwsChunkedV4PayloadSigner.java @@ -69,8 +69,8 @@ private AwsChunkedV4PayloadSigner(Builder builder) { this.credentialScope = Validate.paramNotNull(builder.credentialScope, "CredentialScope"); this.chunkSize = Validate.isPositive(builder.chunkSize, "ChunkSize"); this.checksumAlgorithm = builder.checksumAlgorithm; - this.payloadChecksumStore = builder.payloadChecksumStore == null ? NoOpPayloadChecksumStore.create() : - builder.payloadChecksumStore; + this.payloadChecksumStore = builder.checksumStore == null ? NoOpPayloadChecksumStore.create() : + builder.checksumStore; } public static Builder builder() { @@ -301,7 +301,7 @@ static class Builder { private CredentialScope credentialScope; private Integer chunkSize; private ChecksumAlgorithm checksumAlgorithm; - private PayloadChecksumStore payloadChecksumStore; + private PayloadChecksumStore checksumStore; public Builder credentialScope(CredentialScope credentialScope) { this.credentialScope = credentialScope; @@ -318,8 +318,8 @@ public Builder checksumAlgorithm(ChecksumAlgorithm checksumAlgorithm) { return this; } - public Builder checksumCache(PayloadChecksumStore payloadChecksumStore) { - this.payloadChecksumStore = payloadChecksumStore; + public Builder checksumStore(PayloadChecksumStore checksumStore) { + this.checksumStore = checksumStore; return this; } diff --git a/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/signer/DefaultAwsV4HttpSigner.java b/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/signer/DefaultAwsV4HttpSigner.java index 3865fe4b714b..66ee6d4cf733 100644 --- a/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/signer/DefaultAwsV4HttpSigner.java +++ b/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/signer/DefaultAwsV4HttpSigner.java @@ -24,7 +24,7 @@ import static software.amazon.awssdk.http.auth.aws.internal.signer.util.OptionalDependencyLoaderUtil.getEventStreamV4PayloadSigner; import static software.amazon.awssdk.http.auth.aws.internal.signer.util.SignerConstant.PRESIGN_URL_MAX_EXPIRATION_DURATION; import static software.amazon.awssdk.http.auth.aws.internal.signer.util.SignerConstant.X_AMZ_TRAILER; -import static software.amazon.awssdk.http.auth.spi.signer.SdkInternalHttpSignerProperty.CHECKSUM_CACHE; +import static software.amazon.awssdk.http.auth.spi.signer.SdkInternalHttpSignerProperty.CHECKSUM_STORE; import java.time.Clock; import java.time.Duration; @@ -57,7 +57,7 @@ public final class DefaultAwsV4HttpSigner implements AwsV4HttpSigner { @Override public SignedRequest sign(SignRequest request) { - Checksummer checksummer = checksummer(request, null, checksumCache(request)); + Checksummer checksummer = checksummer(request, null, checksumStore(request)); V4Properties v4Properties = v4Properties(request); V4RequestSigner v4RequestSigner = v4RequestSigner(request, v4Properties); V4PayloadSigner payloadSigner = v4PayloadSigner(request, v4Properties); @@ -170,7 +170,7 @@ private static V4PayloadSigner v4PayloadSigner( return AwsChunkedV4PayloadSigner.builder() .credentialScope(properties.getCredentialScope()) .chunkSize(DEFAULT_CHUNK_SIZE_IN_BYTES) - .checksumCache(checksumCache(request)) + .checksumStore(checksumStore(request)) .checksumAlgorithm(request.property(CHECKSUM_ALGORITHM)) .build(); } @@ -265,8 +265,8 @@ private static boolean isBetweenInclusive(Duration start, Duration x, Duration e return start.compareTo(x) <= 0 && x.compareTo(end) <= 0; } - private static PayloadChecksumStore checksumCache(SignRequest request) { - PayloadChecksumStore cache = request.property(CHECKSUM_CACHE); + private static PayloadChecksumStore checksumStore(SignRequest request) { + PayloadChecksumStore cache = request.property(CHECKSUM_STORE); if (cache == null) { return NoOpPayloadChecksumStore.create(); } diff --git a/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/signer/chunkedencoding/ChecksumTrailerProvider.java b/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/signer/chunkedencoding/ChecksumTrailerProvider.java index 4565a61a3933..dbe4d7c856b0 100644 --- a/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/signer/chunkedencoding/ChecksumTrailerProvider.java +++ b/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/signer/chunkedencoding/ChecksumTrailerProvider.java @@ -30,14 +30,14 @@ public class ChecksumTrailerProvider implements TrailerProvider { private final SdkChecksum checksum; private final String checksumName; private final ChecksumAlgorithm checksumAlgorithm; - private final PayloadChecksumStore checksumCache; + private final PayloadChecksumStore checksumStore; public ChecksumTrailerProvider(SdkChecksum checksum, String checksumName, ChecksumAlgorithm checksumAlgorithm, - PayloadChecksumStore checksumCache) { + PayloadChecksumStore checksumStore) { this.checksum = checksum; this.checksumName = checksumName; this.checksumAlgorithm = checksumAlgorithm; - this.checksumCache = checksumCache; + this.checksumStore = checksumStore; } @Override @@ -47,10 +47,10 @@ public void reset() { @Override public Pair> get() { - byte[] checksumBytes = checksumCache.getChecksumValue(checksumAlgorithm); + byte[] checksumBytes = checksumStore.getChecksumValue(checksumAlgorithm); if (checksumBytes == null) { checksumBytes = checksum.getChecksumBytes(); - checksumCache.putChecksumValue(checksumAlgorithm, checksumBytes); + checksumStore.putChecksumValue(checksumAlgorithm, checksumBytes); } return Pair.of( diff --git a/core/http-auth-aws/src/test/java/software/amazon/awssdk/http/auth/aws/crt/internal/signer/DefaultAwsCrtV4aHttpSignerTest.java b/core/http-auth-aws/src/test/java/software/amazon/awssdk/http/auth/aws/crt/internal/signer/DefaultAwsCrtV4aHttpSignerTest.java index 11867d613380..e1036e7e7cf9 100644 --- a/core/http-auth-aws/src/test/java/software/amazon/awssdk/http/auth/aws/crt/internal/signer/DefaultAwsCrtV4aHttpSignerTest.java +++ b/core/http-auth-aws/src/test/java/software/amazon/awssdk/http/auth/aws/crt/internal/signer/DefaultAwsCrtV4aHttpSignerTest.java @@ -44,7 +44,7 @@ import static software.amazon.awssdk.http.auth.aws.signer.AwsV4aHttpSigner.PAYLOAD_SIGNING_ENABLED; import static software.amazon.awssdk.http.auth.aws.signer.AwsV4aHttpSigner.REGION_SET; import static software.amazon.awssdk.http.auth.spi.signer.HttpSigner.SIGNING_CLOCK; -import static software.amazon.awssdk.http.auth.spi.signer.SdkInternalHttpSignerProperty.CHECKSUM_CACHE; +import static software.amazon.awssdk.http.auth.spi.signer.SdkInternalHttpSignerProperty.CHECKSUM_STORE; import java.io.ByteArrayInputStream; import java.io.IOException; @@ -461,7 +461,7 @@ void sign_WithPayloadSigningTrue_chunkEncodingFalse_cacheContainsChecksum_usesCa signRequest -> signRequest .putProperty(PAYLOAD_SIGNING_ENABLED, true) .putProperty(CHUNK_ENCODING_ENABLED, false) - .putProperty(CHECKSUM_CACHE, cache) + .putProperty(CHECKSUM_STORE, cache) ); SignedRequest signedRequest = signer.sign(request); @@ -480,7 +480,7 @@ void sign_WithPayloadSigningTrue_chunkEncodingFalse_cacheEmpty_storesComputedChe signRequest -> signRequest .putProperty(PAYLOAD_SIGNING_ENABLED, true) .putProperty(CHUNK_ENCODING_ENABLED, false) - .putProperty(CHECKSUM_CACHE, cache) + .putProperty(CHECKSUM_STORE, cache) ); SignedRequest signedRequest = signer.sign(request); @@ -502,7 +502,7 @@ void sign_WithPayloadSigningFalse_chunkEncodingTrue_cacheEmpty_storesComputedChe .putProperty(PAYLOAD_SIGNING_ENABLED, false) .putProperty(CHUNK_ENCODING_ENABLED, true) .putProperty(CHECKSUM_ALGORITHM, CRC32) - .putProperty(CHECKSUM_CACHE, cache) + .putProperty(CHECKSUM_STORE, cache) ); SignedRequest signedRequest = signer.sign(request); @@ -529,7 +529,7 @@ void sign_WithPayloadSigningFalse_chunkEncodingTrue_cacheContainsChecksum_usesCa .putProperty(PAYLOAD_SIGNING_ENABLED, false) .putProperty(CHUNK_ENCODING_ENABLED, true) .putProperty(CHECKSUM_ALGORITHM, CRC32) - .putProperty(CHECKSUM_CACHE, cache) + .putProperty(CHECKSUM_STORE, cache) ); SignedRequest signedRequest = signer.sign(request); @@ -553,7 +553,7 @@ void sign_withPayloadSigningTrue_chunkEncodingFalse_withChecksum_cacheContainsCr .putProperty(PAYLOAD_SIGNING_ENABLED, true) .putProperty(CHUNK_ENCODING_ENABLED, false) .putProperty(CHECKSUM_ALGORITHM, CRC32) - .putProperty(CHECKSUM_CACHE, cache) + .putProperty(CHECKSUM_STORE, cache) ); SignedRequest signedRequest = signer.sign(request); @@ -574,7 +574,7 @@ void sign_withPayloadSigningTrue_chunkEncodingFalse_withChecksum_cacheEmpty_stor .putProperty(PAYLOAD_SIGNING_ENABLED, true) .putProperty(CHUNK_ENCODING_ENABLED, false) .putProperty(CHECKSUM_ALGORITHM, CRC32) - .putProperty(CHECKSUM_CACHE, cache) + .putProperty(CHECKSUM_STORE, cache) ); signer.sign(request); diff --git a/core/http-auth-aws/src/test/java/software/amazon/awssdk/http/auth/aws/internal/signer/DefaultAwsV4HttpSignerTest.java b/core/http-auth-aws/src/test/java/software/amazon/awssdk/http/auth/aws/internal/signer/DefaultAwsV4HttpSignerTest.java index 75c59b40e934..3c4ea21cce58 100644 --- a/core/http-auth-aws/src/test/java/software/amazon/awssdk/http/auth/aws/internal/signer/DefaultAwsV4HttpSignerTest.java +++ b/core/http-auth-aws/src/test/java/software/amazon/awssdk/http/auth/aws/internal/signer/DefaultAwsV4HttpSignerTest.java @@ -28,7 +28,7 @@ import static software.amazon.awssdk.http.auth.aws.signer.AwsV4HttpSigner.CHUNK_ENCODING_ENABLED; import static software.amazon.awssdk.http.auth.aws.signer.AwsV4HttpSigner.EXPIRATION_DURATION; import static software.amazon.awssdk.http.auth.aws.signer.AwsV4HttpSigner.PAYLOAD_SIGNING_ENABLED; -import static software.amazon.awssdk.http.auth.spi.signer.SdkInternalHttpSignerProperty.CHECKSUM_CACHE; +import static software.amazon.awssdk.http.auth.spi.signer.SdkInternalHttpSignerProperty.CHECKSUM_STORE; import java.io.IOException; import java.net.URI; @@ -882,7 +882,7 @@ void sign_WithPayloadSigningTrue_chunkEncodingFalse_cacheContainsChecksum_usesCa signRequest -> signRequest .putProperty(PAYLOAD_SIGNING_ENABLED, true) .putProperty(CHUNK_ENCODING_ENABLED, false) - .putProperty(CHECKSUM_CACHE, cache) + .putProperty(CHECKSUM_STORE, cache) ); SignedRequest signedRequest = signer.sign(request); @@ -901,7 +901,7 @@ void sign_WithPayloadSigningTrue_chunkEncodingFalse_cacheEmpty_storesComputedChe signRequest -> signRequest .putProperty(PAYLOAD_SIGNING_ENABLED, true) .putProperty(CHUNK_ENCODING_ENABLED, false) - .putProperty(CHECKSUM_CACHE, cache) + .putProperty(CHECKSUM_STORE, cache) ); SignedRequest signedRequest = signer.sign(request); @@ -923,7 +923,7 @@ void sign_WithPayloadSigningFalse_chunkEncodingTrue_cacheEmpty_storesComputedChe .putProperty(PAYLOAD_SIGNING_ENABLED, false) .putProperty(CHUNK_ENCODING_ENABLED, true) .putProperty(CHECKSUM_ALGORITHM, CRC32) - .putProperty(CHECKSUM_CACHE, cache) + .putProperty(CHECKSUM_STORE, cache) ); SignedRequest signedRequest = signer.sign(request); @@ -950,7 +950,7 @@ void sign_WithPayloadSigningFalse_chunkEncodingTrue_cacheContainsChecksum_usesCa .putProperty(PAYLOAD_SIGNING_ENABLED, false) .putProperty(CHUNK_ENCODING_ENABLED, true) .putProperty(CHECKSUM_ALGORITHM, CRC32) - .putProperty(CHECKSUM_CACHE, cache) + .putProperty(CHECKSUM_STORE, cache) ); SignedRequest signedRequest = signer.sign(request); @@ -977,7 +977,7 @@ void sign_withPayloadSigningTrue_chunkEncodingFalse_withChecksum_cacheContainsCr .putProperty(PAYLOAD_SIGNING_ENABLED, true) .putProperty(CHUNK_ENCODING_ENABLED, false) .putProperty(CHECKSUM_ALGORITHM, CRC32) - .putProperty(CHECKSUM_CACHE, cache) + .putProperty(CHECKSUM_STORE, cache) ); SignedRequest signedRequest = signer.sign(request); @@ -998,7 +998,7 @@ void sign_withPayloadSigningTrue_chunkEncodingFalse_withChecksum_cacheEmpty_stor .putProperty(PAYLOAD_SIGNING_ENABLED, true) .putProperty(CHUNK_ENCODING_ENABLED, false) .putProperty(CHECKSUM_ALGORITHM, CRC32) - .putProperty(CHECKSUM_CACHE, cache) + .putProperty(CHECKSUM_STORE, cache) ); signer.sign(request); diff --git a/core/http-auth-spi/src/main/java/software/amazon/awssdk/http/auth/spi/signer/SdkInternalHttpSignerProperty.java b/core/http-auth-spi/src/main/java/software/amazon/awssdk/http/auth/spi/signer/SdkInternalHttpSignerProperty.java index bb4a4c3310bf..d95db35f6e46 100644 --- a/core/http-auth-spi/src/main/java/software/amazon/awssdk/http/auth/spi/signer/SdkInternalHttpSignerProperty.java +++ b/core/http-auth-spi/src/main/java/software/amazon/awssdk/http/auth/spi/signer/SdkInternalHttpSignerProperty.java @@ -25,12 +25,12 @@ public final class SdkInternalHttpSignerProperty { /** - * A cache for storing checksums calculated for a payload. + * An object for storing checksums calculated for a payload. * *

Note, checksums may not be relevant to some signers. */ - public static final SignerProperty CHECKSUM_CACHE = - SignerProperty.create(SdkInternalHttpSignerProperty.class, "ChecksumCache"); + public static final SignerProperty CHECKSUM_STORE = + SignerProperty.create(SdkInternalHttpSignerProperty.class, "ChecksumStore"); private SdkInternalHttpSignerProperty() { } diff --git a/core/sdk-core/src/main/java/software/amazon/awssdk/core/interceptor/SdkInternalExecutionAttribute.java b/core/sdk-core/src/main/java/software/amazon/awssdk/core/interceptor/SdkInternalExecutionAttribute.java index 80e3142c7022..3fe0f69d3ab8 100644 --- a/core/sdk-core/src/main/java/software/amazon/awssdk/core/interceptor/SdkInternalExecutionAttribute.java +++ b/core/sdk-core/src/main/java/software/amazon/awssdk/core/interceptor/SdkInternalExecutionAttribute.java @@ -207,10 +207,10 @@ public final class SdkInternalExecutionAttribute extends SdkExecutionAttribute { "TokenConfiguredFromEnv"); /** - * The cache used by {@link HttpSigner} implementations to store payload checksums. + * The store used by {@link HttpSigner} implementations to store payload checksums. */ - public static final ExecutionAttribute CHECKSUM_CACHE = - new ExecutionAttribute<>("ChecksumCache"); + public static final ExecutionAttribute CHECKSUM_STORE = + new ExecutionAttribute<>("ChecksumStore"); /** * The backing attribute for RESOLVED_CHECKSUM_SPECS. diff --git a/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/async/ChecksumCalculatingAsyncRequestBody.java b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/async/ChecksumCalculatingAsyncRequestBody.java index 2226c67fcdc9..f7d61de1353e 100644 --- a/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/async/ChecksumCalculatingAsyncRequestBody.java +++ b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/async/ChecksumCalculatingAsyncRequestBody.java @@ -35,6 +35,8 @@ import software.amazon.awssdk.core.checksums.Algorithm; import software.amazon.awssdk.core.exception.SdkException; import software.amazon.awssdk.core.internal.util.HttpChecksumUtils; +import software.amazon.awssdk.http.auth.aws.internal.signer.NoOpPayloadChecksumStore; +import software.amazon.awssdk.http.auth.spi.signer.PayloadChecksumStore; import software.amazon.awssdk.utils.BinaryUtils; import software.amazon.awssdk.utils.Validate; import software.amazon.awssdk.utils.async.DelegatingSubscriber; @@ -55,6 +57,7 @@ public class ChecksumCalculatingAsyncRequestBody implements AsyncRequestBody { private final ChecksumAlgorithm algorithm; private final String trailerHeader; private final long totalBytes; + private final PayloadChecksumStore payloadChecksumStore; private ChecksumCalculatingAsyncRequestBody(DefaultBuilder builder) { @@ -66,6 +69,7 @@ private ChecksumCalculatingAsyncRequestBody(DefaultBuilder builder) { this.sdkChecksum = builder.algorithm != null ? SdkChecksum.forAlgorithm(algorithm) : null; this.trailerHeader = builder.trailerHeader; this.totalBytes = initTotalBytes(wrapped, builder.contentLengthHeader); + this.payloadChecksumStore = builder.checksumStore != null ? builder.checksumStore : NoOpPayloadChecksumStore.create(); } static long initTotalBytes(AsyncRequestBody wrapped, Long contentLengthHeader) { @@ -118,6 +122,8 @@ public interface Builder extends SdkBuilder s) { SynchronousChunkBuffer synchronousChunkBuffer = new SynchronousChunkBuffer(totalBytes); alwaysInvokeOnNext(wrapped.flatMapIterable(synchronousChunkBuffer::buffer)) - .subscribe(new ChecksumCalculatingSubscriber(s, sdkChecksum, trailerHeader, totalBytes)); + .subscribe(new ChecksumCalculatingSubscriber(s, + algorithm, + sdkChecksum, + payloadChecksumStore, + trailerHeader, + totalBytes)); } private SdkPublisher alwaysInvokeOnNext(SdkPublisher source) { @@ -192,17 +210,24 @@ private SdkPublisher alwaysInvokeOnNext(SdkPublisher sou private static final class ChecksumCalculatingSubscriber implements Subscriber { private final Subscriber wrapped; + private final ChecksumAlgorithm algorithm; private final SdkChecksum checksum; + private final PayloadChecksumStore checksumStore; private final String trailerHeader; private byte[] checksumBytes; private final AtomicLong remainingBytes; private Subscription subscription; ChecksumCalculatingSubscriber(Subscriber wrapped, + ChecksumAlgorithm algorithm, SdkChecksum checksum, - String trailerHeader, long totalBytes) { + PayloadChecksumStore checksumStore, + String trailerHeader, + long totalBytes) { this.wrapped = wrapped; + this.algorithm = algorithm; this.checksum = checksum; + this.checksumStore = checksumStore; this.trailerHeader = trailerHeader; this.remainingBytes = new AtomicLong(totalBytes); } @@ -223,7 +248,11 @@ public void onNext(ByteBuffer byteBuffer) { byteBuffer.reset(); } if (lastByte && checksumBytes == null && checksum != null) { - checksumBytes = checksum.getChecksumBytes(); + checksumBytes = checksumStore.getChecksumValue(algorithm); + if (checksumBytes == null) { + checksumBytes = checksum.getChecksumBytes(); + checksumStore.putChecksumValue(algorithm, checksumBytes); + } ByteBuffer allocatedBuffer = getFinalChecksumAppendedChunk(byteBuffer); wrapped.onNext(allocatedBuffer); } else if (byteBuffer.hasRemaining()) { diff --git a/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/http/pipeline/stages/HttpChecksumStage.java b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/http/pipeline/stages/HttpChecksumStage.java index 6f958cc975e9..b2925f55c3d3 100644 --- a/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/http/pipeline/stages/HttpChecksumStage.java +++ b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/http/pipeline/stages/HttpChecksumStage.java @@ -22,6 +22,7 @@ import static software.amazon.awssdk.core.HttpChecksumConstant.SIGNING_METHOD; import static software.amazon.awssdk.core.interceptor.SdkExecutionAttribute.RESOLVED_CHECKSUM_SPECS; import static software.amazon.awssdk.core.interceptor.SdkInternalExecutionAttribute.AUTH_SCHEMES; +import static software.amazon.awssdk.core.interceptor.SdkInternalExecutionAttribute.CHECKSUM_STORE; import static software.amazon.awssdk.core.internal.io.AwsChunkedInputStream.DEFAULT_CHUNK_SIZE; import static software.amazon.awssdk.core.internal.util.ChunkContentUtils.calculateChecksumTrailerLength; import static software.amazon.awssdk.core.internal.util.ChunkContentUtils.calculateStreamContentLength; @@ -77,7 +78,7 @@ public SdkHttpFullRequest.Builder execute(SdkHttpFullRequest.Builder request, Re throws Exception { if (sraSigningEnabled(context)) { - ensurePayloadChecksumCachePresent(context.executionAttributes()); + ensurePayloadChecksumStorePresent(context.executionAttributes()); return sraChecksum(request, context); } @@ -246,6 +247,7 @@ private void addFlexibleChecksumInTrailer(SdkHttpFullRequest.Builder request, Re ChecksumCalculatingAsyncRequestBody.builder() .asyncRequestBody(context.requestProvider()) .algorithm(checksumSpecs.algorithmV2()) + .checksumStore(getPayloadChecksumStore(context.executionAttributes())) .trailerHeader(checksumSpecs.headerName()); Optional maybeContentLengthHeader = request.firstMatchingHeader("Content-Length") .map(Long::parseLong); @@ -321,14 +323,18 @@ private void addFlexibleChecksumInHeader(SdkHttpFullRequest.Builder request, Req } } - private void ensurePayloadChecksumCachePresent(ExecutionAttributes executionAttributes) { - PayloadChecksumStore cache = executionAttributes.getAttribute(SdkInternalExecutionAttribute.CHECKSUM_CACHE); + private void ensurePayloadChecksumStorePresent(ExecutionAttributes executionAttributes) { + PayloadChecksumStore cache = getPayloadChecksumStore(executionAttributes); if (cache == null) { cache = PayloadChecksumStore.create(); - executionAttributes.putAttribute(SdkInternalExecutionAttribute.CHECKSUM_CACHE, cache); + executionAttributes.putAttribute(CHECKSUM_STORE, cache); } } + private PayloadChecksumStore getPayloadChecksumStore(ExecutionAttributes executionAttributes) { + return executionAttributes.getAttribute(CHECKSUM_STORE); + } + static final class ChecksumCalculatingStreamProvider implements ContentStreamProvider { private final ContentStreamProvider underlyingInputStreamProvider; private final String checksumHeaderForTrailer; diff --git a/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/http/pipeline/stages/SigningStage.java b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/http/pipeline/stages/SigningStage.java index d9b65f43f386..3d3240c3d4d8 100644 --- a/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/http/pipeline/stages/SigningStage.java +++ b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/http/pipeline/stages/SigningStage.java @@ -93,7 +93,7 @@ private SdkHttpFullRequest sraSignRequest(SdkHttpFullReques // Should not be null, added by HttpChecksumStage for SRA signed requests PayloadChecksumStore payloadChecksumStore = - context.executionAttributes().getAttribute(SdkInternalExecutionAttribute.CHECKSUM_CACHE); + context.executionAttributes().getAttribute(SdkInternalExecutionAttribute.CHECKSUM_STORE); Pair measuredSign = MetricUtils.measureDuration( () -> doSraSign(request, selectedAuthScheme, identity, payloadChecksumStore)); @@ -111,7 +111,7 @@ private SdkHttpFullRequest doSraSign(SdkHttpFullRequest req SignRequest.Builder signRequestBuilder = SignRequest .builder(identity) .putProperty(HttpSigner.SIGNING_CLOCK, signingClock()) - .putProperty(SdkInternalHttpSignerProperty.CHECKSUM_CACHE, payloadChecksumStore) + .putProperty(SdkInternalHttpSignerProperty.CHECKSUM_STORE, payloadChecksumStore) .request(request) .payload(request.contentStreamProvider().orElse(null)); AuthSchemeOption authSchemeOption = selectedAuthScheme.authSchemeOption(); diff --git a/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/async/ChecksumCalculatingAsyncRequestBodyTest.java b/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/async/ChecksumCalculatingAsyncRequestBodyTest.java index eddc94da63ed..c17d2c3fdaab 100644 --- a/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/async/ChecksumCalculatingAsyncRequestBodyTest.java +++ b/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/async/ChecksumCalculatingAsyncRequestBodyTest.java @@ -15,6 +15,7 @@ package software.amazon.awssdk.core.internal.async; +import java.io.ByteArrayOutputStream; import java.io.IOException; import java.nio.ByteBuffer; import java.nio.charset.StandardCharsets; @@ -42,6 +43,7 @@ import software.amazon.awssdk.core.async.AsyncRequestBody; import software.amazon.awssdk.core.internal.util.Mimetype; import software.amazon.awssdk.http.async.SimpleSubscriber; +import software.amazon.awssdk.http.auth.spi.signer.PayloadChecksumStore; import software.amazon.awssdk.utils.BinaryUtils; import static org.assertj.core.api.Assertions.assertThat; @@ -301,6 +303,61 @@ public void explicit0ContentLength_containsEmptyStringTrailingChecksum(TestCase assertThat(sb.toString()).isEqualTo(expectedEmptyString); } + @Test + void subscribe_checksumStoreContainsChecksumValue_reusesValue() { + byte[] content = "Hello world".getBytes(StandardCharsets.UTF_8); + AsyncRequestBody body = AsyncRequestBody.fromBytes(content); + + byte[] checksumValue = "my-checksum".getBytes(StandardCharsets.UTF_8); + PayloadChecksumStore store = PayloadChecksumStore.create(); + store.putChecksumValue(DefaultChecksumAlgorithm.CRC32, checksumValue); + + String trailerHeader = "x-amz-checksum-crc32"; + ChecksumCalculatingAsyncRequestBody checksumBody = + ChecksumCalculatingAsyncRequestBody.builder() + .contentLengthHeader((long) content.length) + .trailerHeader(trailerHeader) + .algorithm(DefaultChecksumAlgorithm.CRC32) + .checksumStore(store) + .asyncRequestBody(body) + .build(); + + String encoded = toString(checksumBody); + + assertThat(encoded).endsWith(String.format("%s:%s\r\n\r\n", trailerHeader, BinaryUtils.toBase64(checksumValue))); + } + + @Test + void subscribe_checksumStoreEmpty_storesComputedValue() { + byte[] content = "Hello world".getBytes(StandardCharsets.UTF_8); + AsyncRequestBody body = AsyncRequestBody.fromBytes(content); + + String expectedChecksum = "i9aeUg=="; + + PayloadChecksumStore store = PayloadChecksumStore.create(); + + String trailerHeader = "x-amz-checksum-crc32"; + ChecksumCalculatingAsyncRequestBody checksumBody = + ChecksumCalculatingAsyncRequestBody.builder() + .contentLengthHeader((long) content.length) + .trailerHeader(trailerHeader) + .algorithm(DefaultChecksumAlgorithm.CRC32) + .checksumStore(store) + .asyncRequestBody(body) + .build(); + + String encoded = toString(checksumBody); + + assertThat(encoded).endsWith(String.format("%s:%s\r\n\r\n", trailerHeader, expectedChecksum)); + assertThat(store.getChecksumValue(DefaultChecksumAlgorithm.CRC32)).isEqualTo(BinaryUtils.fromBase64(expectedChecksum)); + } + + private static String toString(Publisher publisher) { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + Flowable.fromPublisher(publisher).blockingForEach(chunk -> baos.write(BinaryUtils.copyAllBytesFrom(chunk))); + return new String(baos.toByteArray(), StandardCharsets.UTF_8); + } + static class EmptyBufferPublisher implements AsyncRequestBody { private final ByteBuffer[] buffers = new ByteBuffer[2]; diff --git a/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/http/pipeline/stages/HttpChecksumStageSraTest.java b/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/http/pipeline/stages/HttpChecksumStageSraTest.java index 65d778839f4f..ed166597a441 100644 --- a/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/http/pipeline/stages/HttpChecksumStageSraTest.java +++ b/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/http/pipeline/stages/HttpChecksumStageSraTest.java @@ -20,7 +20,7 @@ import static software.amazon.awssdk.core.HttpChecksumConstant.SIGNING_METHOD; import static software.amazon.awssdk.core.interceptor.SdkExecutionAttribute.RESOLVED_CHECKSUM_SPECS; import static software.amazon.awssdk.core.interceptor.SdkInternalExecutionAttribute.AUTH_SCHEMES; -import static software.amazon.awssdk.core.interceptor.SdkInternalExecutionAttribute.CHECKSUM_CACHE; +import static software.amazon.awssdk.core.interceptor.SdkInternalExecutionAttribute.CHECKSUM_STORE; import static software.amazon.awssdk.core.internal.signer.SigningMethod.UNSIGNED_PAYLOAD; import static software.amazon.awssdk.http.Header.CONTENT_LENGTH; import static software.amazon.awssdk.http.Header.CONTENT_MD5; @@ -146,28 +146,28 @@ public void async_flexibleChecksumInTrailer_addsFlexibleChecksumInTrailer() thro } @Test - public void execute_checksumCacheAttributeNotPresent_shouldCreate() throws Exception { + public void execute_checksumStoreAttributeNotPresent_shouldCreate() throws Exception { SdkHttpFullRequest.Builder requestBuilder = createHttpRequestBuilder(); RequestExecutionContext ctx = flexibleChecksumRequestContext(ClientType.SYNC, ChecksumSpecs.builder().isRequestChecksumRequired(true), false); new HttpChecksumStage(ClientType.SYNC).execute(requestBuilder, ctx); - assertThat(ctx.executionAttributes().getAttribute(CHECKSUM_CACHE)).isNotNull(); + assertThat(ctx.executionAttributes().getAttribute(CHECKSUM_STORE)).isNotNull(); } @Test - public void execute_checksumCacheAttributePresent_shouldNotOverwrite() throws Exception { + public void execute_checksumStoreAttributePresent_shouldNotOverwrite() throws Exception { PayloadChecksumStore cache = PayloadChecksumStore.create(); SdkHttpFullRequest.Builder requestBuilder = createHttpRequestBuilder(); RequestExecutionContext ctx = flexibleChecksumRequestContext(ClientType.SYNC, ChecksumSpecs.builder().isRequestChecksumRequired(true), false); - ctx.executionAttributes().putAttribute(CHECKSUM_CACHE, cache); + ctx.executionAttributes().putAttribute(CHECKSUM_STORE, cache); new HttpChecksumStage(ClientType.SYNC).execute(requestBuilder, ctx); - assertThat(ctx.executionAttributes().getAttribute(CHECKSUM_CACHE)).isSameAs(cache); + assertThat(ctx.executionAttributes().getAttribute(CHECKSUM_STORE)).isSameAs(cache); } private SdkHttpFullRequest.Builder createHttpRequestBuilder() { diff --git a/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/http/pipeline/stages/SigningStageTest.java b/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/http/pipeline/stages/SigningStageTest.java index b0d87b547bd9..d86ba2c70e5b 100644 --- a/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/http/pipeline/stages/SigningStageTest.java +++ b/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/http/pipeline/stages/SigningStageTest.java @@ -23,7 +23,7 @@ import static org.mockito.Mockito.verifyNoInteractions; import static org.mockito.Mockito.when; import static software.amazon.awssdk.core.interceptor.SdkExecutionAttribute.TIME_OFFSET; -import static software.amazon.awssdk.core.interceptor.SdkInternalExecutionAttribute.CHECKSUM_CACHE; +import static software.amazon.awssdk.core.interceptor.SdkInternalExecutionAttribute.CHECKSUM_STORE; import static software.amazon.awssdk.core.interceptor.SdkInternalExecutionAttribute.SELECTED_AUTH_SCHEME; import static software.amazon.awssdk.core.metrics.CoreMetric.SIGNING_DURATION; @@ -376,7 +376,7 @@ public void execute_selectedAuthScheme_signer_doesPreSraSign() throws Exception } @Test - public void execute_checksumCacheAttributePresent_propagatesChecksumCacheToSigner() throws Exception { + public void execute_checksumStoreAttributePresent_propagatesChecksumStoreToSigner() throws Exception { SelectedAuthScheme selectedAuthScheme = new SelectedAuthScheme<>( CompletableFuture.completedFuture(identity), httpSigner, @@ -387,7 +387,7 @@ public void execute_checksumCacheAttributePresent_propagatesChecksumCacheToSigne RequestExecutionContext context = createContext(selectedAuthScheme, null); PayloadChecksumStore cache = PayloadChecksumStore.create(); - context.executionAttributes().putAttribute(CHECKSUM_CACHE, cache); + context.executionAttributes().putAttribute(CHECKSUM_STORE, cache); SdkHttpRequest signedRequest = ValidSdkObjects.sdkHttpFullRequest().build(); when(httpSigner.sign(ArgumentMatchers.>any())) @@ -401,7 +401,7 @@ public void execute_checksumCacheAttributePresent_propagatesChecksumCacheToSigne ArgumentCaptor> signRequestCaptor = ArgumentCaptor.forClass(SignRequest.class); verify(httpSigner).sign(signRequestCaptor.capture()); - assertThat(signRequestCaptor.getValue().property(SdkInternalHttpSignerProperty.CHECKSUM_CACHE)).isSameAs(cache); + assertThat(signRequestCaptor.getValue().property(SdkInternalHttpSignerProperty.CHECKSUM_STORE)).isSameAs(cache); } private RequestExecutionContext createContext(SelectedAuthScheme selectedAuthScheme, Signer oldSigner) { diff --git a/services/s3/src/test/java/software/amazon/awssdk/services/s3/checksums/ChecksumReuseTest.java b/services/s3/src/test/java/software/amazon/awssdk/services/s3/checksums/ChecksumReuseTest.java index 6b30d935087f..4158945ef302 100644 --- a/services/s3/src/test/java/software/amazon/awssdk/services/s3/checksums/ChecksumReuseTest.java +++ b/services/s3/src/test/java/software/amazon/awssdk/services/s3/checksums/ChecksumReuseTest.java @@ -18,17 +18,26 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; +import io.reactivex.Flowable; import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.InputStream; +import java.nio.ByteBuffer; import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.List; import java.util.Random; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider; import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; +import software.amazon.awssdk.core.async.AsyncRequestBody; import software.amazon.awssdk.core.checksums.RequestChecksumCalculation; import software.amazon.awssdk.core.exception.SdkException; import software.amazon.awssdk.core.sync.RequestBody; @@ -38,12 +47,19 @@ import software.amazon.awssdk.http.HttpExecuteResponse; import software.amazon.awssdk.http.SdkHttpClient; import software.amazon.awssdk.http.SdkHttpFullResponse; +import software.amazon.awssdk.http.async.AsyncExecuteRequest; +import software.amazon.awssdk.http.async.SdkAsyncHttpClient; +import software.amazon.awssdk.http.async.SdkAsyncHttpResponseHandler; +import software.amazon.awssdk.http.async.SdkHttpContentPublisher; import software.amazon.awssdk.regions.Region; import software.amazon.awssdk.retries.StandardRetryStrategy; import software.amazon.awssdk.retries.api.BackoffStrategy; +import software.amazon.awssdk.services.s3.S3AsyncClient; import software.amazon.awssdk.services.s3.S3Client; import software.amazon.awssdk.services.s3.model.ChecksumAlgorithm; +import software.amazon.awssdk.services.s3.model.PutObjectResponse; import software.amazon.awssdk.services.s3.model.S3Exception; +import software.amazon.awssdk.utils.BinaryUtils; import software.amazon.awssdk.utils.IoUtils; /** @@ -55,6 +71,18 @@ public class ChecksumReuseTest { private static final AwsCredentialsProvider CREDENTIALS_PROVIDER = StaticCredentialsProvider.create( AwsBasicCredentials.create("akid", "skid")); + private static ExecutorService executorService; + + @BeforeAll + public static void setup() { + executorService = Executors.newSingleThreadExecutor(); + } + + @AfterAll + public static void teardown() { + executorService.shutdownNow(); + } + @Test public void putObject_serverResponds500_usesSameChecksumOnRetries() { MockHttpClient httpClient = new MockHttpClient(); @@ -78,11 +106,45 @@ public void putObject_serverResponds500_usesSameChecksumOnRetries() { // Ensure we actually retried .matches(e -> ((SdkException) e).numAttempts() == 4); + assertAllTrailingChecksumsMatch(httpClient.requestPayloads); + } + + @Test + void asyncPutObject_serverResponds500_usesSameChecksumOnRetries() { + MockAsyncHttpClient httpClient = new MockAsyncHttpClient(); + + S3AsyncClient s3 = S3AsyncClient.builder() + .region(Region.US_WEST_2) + .credentialsProvider(CREDENTIALS_PROVIDER) + .requestChecksumCalculation(RequestChecksumCalculation.WHEN_SUPPORTED) + .httpClient(httpClient) + .overrideConfiguration(o -> o.retryStrategy(StandardRetryStrategy.builder() + .maxAttempts(4) + .backoffStrategy(BackoffStrategy.retryImmediately()) + .build())) + .build(); + + AsyncRequestBody requestBody = AsyncRequestBody.fromInputStream(new RandomInputStream(), + 4096L, + executorService); + + CompletableFuture responseFuture = + s3.putObject(r -> r.bucket(BUCKET).key(KEY).checksumAlgorithm(ChecksumAlgorithm.CRC32), requestBody); + + assertThatThrownBy(responseFuture::join) + .hasCauseInstanceOf(S3Exception.class) + .matches(e -> ((SdkException) e.getCause()).numAttempts() == 4); + + assertAllTrailingChecksumsMatch(httpClient.requestPayloads); + } + + private void assertAllTrailingChecksumsMatch(List requestPayloads) { List trailingChecksumHeaders = new ArrayList<>(); - for (byte[] requestPayload : httpClient.requestPayloads) { + + for (byte[] requestPayload : requestPayloads) { String payloadAsString = new String(requestPayload, StandardCharsets.UTF_8); for (String part : payloadAsString.split("\r\n")) { - if (part.startsWith("x-amz-checksum-crc32")) { + if (part.startsWith("x-amz-checksum-crc32:")) { trailingChecksumHeaders.add(part); break; } @@ -122,9 +184,11 @@ public HttpExecuteResponse call() throws IOException { requestPayloads.add(IoUtils.toByteArray(content)); return HttpExecuteResponse.builder() - .response(SdkHttpFullResponse.builder().statusCode(503) + .response(SdkHttpFullResponse.builder() + .statusCode(503) .build()) - .responseBody(AbortableInputStream.create(new ByteArrayInputStream("".getBytes(StandardCharsets.UTF_8)))) + .responseBody(AbortableInputStream.create( + new ByteArrayInputStream("".getBytes(StandardCharsets.UTF_8)))) .build(); } @@ -160,4 +224,36 @@ public synchronized void mark(int readlimit) { public synchronized void reset() throws IOException { } } + + private static class MockAsyncHttpClient implements SdkAsyncHttpClient { + private final List requestPayloads = new ArrayList<>(); + + @Override + public CompletableFuture execute(AsyncExecuteRequest request) { + SdkHttpContentPublisher contentPublisher = request.requestContentPublisher(); + + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + Flowable.fromPublisher(contentPublisher).blockingForEach(bb -> baos.write(BinaryUtils.copyBytesFrom(bb))); + + requestPayloads.add( baos.toByteArray()); + + SdkAsyncHttpResponseHandler responseHandler = request.responseHandler(); + + SdkHttpFullResponse response = SdkHttpFullResponse.builder() + .statusCode(503) + .build(); + + responseHandler.onHeaders(response); + + CompletableFuture future = new CompletableFuture<>(); + responseHandler.onStream(Flowable.just(ByteBuffer.wrap("".getBytes(StandardCharsets.UTF_8))) + .doAfterTerminate(() -> future.complete(null))); + + return future; + } + + @Override + public void close() { + } + } } From 4c45bb4604b91c7ea37d081c42f287d7ae0e701d Mon Sep 17 00:00:00 2001 From: Dongie Agnir Date: Wed, 24 Sep 2025 13:13:44 -0700 Subject: [PATCH 2/2] Fix cross module issue --- .../ChecksumCalculatingAsyncRequestBody.java | 2 +- .../checksums/NoOpPayloadChecksumStore.java | 45 +++++++++++++++++++ 2 files changed, 46 insertions(+), 1 deletion(-) create mode 100644 core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/checksums/NoOpPayloadChecksumStore.java diff --git a/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/async/ChecksumCalculatingAsyncRequestBody.java b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/async/ChecksumCalculatingAsyncRequestBody.java index f7d61de1353e..c903829d07e2 100644 --- a/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/async/ChecksumCalculatingAsyncRequestBody.java +++ b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/async/ChecksumCalculatingAsyncRequestBody.java @@ -34,8 +34,8 @@ import software.amazon.awssdk.core.async.SdkPublisher; import software.amazon.awssdk.core.checksums.Algorithm; import software.amazon.awssdk.core.exception.SdkException; +import software.amazon.awssdk.core.internal.checksums.NoOpPayloadChecksumStore; import software.amazon.awssdk.core.internal.util.HttpChecksumUtils; -import software.amazon.awssdk.http.auth.aws.internal.signer.NoOpPayloadChecksumStore; import software.amazon.awssdk.http.auth.spi.signer.PayloadChecksumStore; import software.amazon.awssdk.utils.BinaryUtils; import software.amazon.awssdk.utils.Validate; diff --git a/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/checksums/NoOpPayloadChecksumStore.java b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/checksums/NoOpPayloadChecksumStore.java new file mode 100644 index 000000000000..68c4b23cc687 --- /dev/null +++ b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/checksums/NoOpPayloadChecksumStore.java @@ -0,0 +1,45 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.core.internal.checksums; + +import software.amazon.awssdk.annotations.SdkInternalApi; +import software.amazon.awssdk.checksums.spi.ChecksumAlgorithm; +import software.amazon.awssdk.http.auth.spi.signer.PayloadChecksumStore; + +@SdkInternalApi +public final class NoOpPayloadChecksumStore implements PayloadChecksumStore { + private NoOpPayloadChecksumStore() { + } + + @Override + public byte[] putChecksumValue(ChecksumAlgorithm algorithm, byte[] checksum) { + return null; + } + + @Override + public byte[] getChecksumValue(ChecksumAlgorithm algorithm) { + return null; + } + + @Override + public boolean containsChecksumValue(ChecksumAlgorithm algorithm) { + return false; + } + + public static NoOpPayloadChecksumStore create() { + return new NoOpPayloadChecksumStore(); + } +}