From ed79e25312438703b344e93b7c907db428cfe00e Mon Sep 17 00:00:00 2001 From: Anirav Kareddy Date: Tue, 22 Jul 2025 14:40:57 -0700 Subject: [PATCH 01/26] using the @RetryingTest annotation for CI: first attempt --- pom.xml | 7 +++++++ .../s3/S3EncryptionClientInstructionFileTest.java | 3 ++- .../amazon/encryption/s3/S3EncryptionClientStreamTest.java | 7 ++++--- 3 files changed, 13 insertions(+), 4 deletions(-) diff --git a/pom.xml b/pom.xml index 331cf5ac9..da75d3026 100644 --- a/pom.xml +++ b/pom.xml @@ -134,6 +134,13 @@ test + + org.junit-pioneer + junit-pioneer + 2.3.0 + test + + com.amazonaws aws-java-sdk-kms diff --git a/src/test/java/software/amazon/encryption/s3/S3EncryptionClientInstructionFileTest.java b/src/test/java/software/amazon/encryption/s3/S3EncryptionClientInstructionFileTest.java index 9ca5b8352..f05433476 100644 --- a/src/test/java/software/amazon/encryption/s3/S3EncryptionClientInstructionFileTest.java +++ b/src/test/java/software/amazon/encryption/s3/S3EncryptionClientInstructionFileTest.java @@ -11,6 +11,7 @@ import com.amazonaws.services.s3.model.StaticEncryptionMaterialsProvider; import org.apache.commons.io.IOUtils; import org.junit.jupiter.api.Test; +import org.junitpioneer.jupiter.RetryingTest; import software.amazon.awssdk.core.ResponseBytes; import software.amazon.awssdk.core.ResponseInputStream; import software.amazon.awssdk.core.sync.RequestBody; @@ -309,7 +310,7 @@ public void testPutWithInstructionFileV3ToV2Rsa() throws NoSuchAlgorithmExceptio s3Client.close(); } - @Test + @RetryingTest(maxAttempts = 3) public void testMultipartPutWithInstructionFile() throws IOException { final String object_key = appendTestSuffix("test-multipart-put-instruction-file"); diff --git a/src/test/java/software/amazon/encryption/s3/S3EncryptionClientStreamTest.java b/src/test/java/software/amazon/encryption/s3/S3EncryptionClientStreamTest.java index b13b886cf..cf0a59d80 100644 --- a/src/test/java/software/amazon/encryption/s3/S3EncryptionClientStreamTest.java +++ b/src/test/java/software/amazon/encryption/s3/S3EncryptionClientStreamTest.java @@ -13,6 +13,7 @@ import org.bouncycastle.jce.provider.BouncyCastleProvider; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; +import org.junitpioneer.jupiter.RetryingTest; import software.amazon.awssdk.core.ResponseBytes; import software.amazon.awssdk.core.ResponseInputStream; import software.amazon.awssdk.core.async.AsyncRequestBody; @@ -306,7 +307,7 @@ public void failsWhenBothBufferSizeAndDelayedAuthModeEnabled() { .build()); } - @Test + @RetryingTest(maxAttempts = 3) public void customSetBufferSizeWithLargeObject() throws IOException { final String objectKey = appendTestSuffix("large-object-test-custom-buffer-size"); @@ -358,7 +359,7 @@ public void customSetBufferSizeWithLargeObject() throws IOException { v3ClientWithDelayedAuth.close(); } - @Test + @RetryingTest(maxAttempts = 3) public void customSetBufferSizeWithLargeObjectAsyncClient() throws IOException { final String objectKey = appendTestSuffix("large-object-test-custom-buffer-size-async"); @@ -418,7 +419,7 @@ public void customSetBufferSizeWithLargeObjectAsyncClient() throws IOException { v3ClientWithDelayedAuth.close(); } - @Test + @RetryingTest(maxAttempts = 3) public void delayedAuthModeWithLargeObject() throws IOException { final String objectKey = appendTestSuffix("large-object-test"); From f128870b77e896e6fe3c9fce028dff27200adcca Mon Sep 17 00:00:00 2001 From: Anirav Kareddy Date: Tue, 22 Jul 2025 14:58:39 -0700 Subject: [PATCH 02/26] could not use the RetryingTest dependency since it relies on Java 11 and we are relying on Java 8. Workaround was using RepeatedTests annotation and add a testCasePassed flag ... let's see if it works --- pom.xml | 7 ---- ...S3EncryptionClientInstructionFileTest.java | 11 +++++-- .../s3/S3EncryptionClientStreamTest.java | 32 ++++++++++++++++--- 3 files changed, 37 insertions(+), 13 deletions(-) diff --git a/pom.xml b/pom.xml index da75d3026..331cf5ac9 100644 --- a/pom.xml +++ b/pom.xml @@ -134,13 +134,6 @@ test - - org.junit-pioneer - junit-pioneer - 2.3.0 - test - - com.amazonaws aws-java-sdk-kms diff --git a/src/test/java/software/amazon/encryption/s3/S3EncryptionClientInstructionFileTest.java b/src/test/java/software/amazon/encryption/s3/S3EncryptionClientInstructionFileTest.java index f05433476..680a6fc23 100644 --- a/src/test/java/software/amazon/encryption/s3/S3EncryptionClientInstructionFileTest.java +++ b/src/test/java/software/amazon/encryption/s3/S3EncryptionClientInstructionFileTest.java @@ -10,8 +10,9 @@ import com.amazonaws.services.s3.model.KMSEncryptionMaterials; import com.amazonaws.services.s3.model.StaticEncryptionMaterialsProvider; import org.apache.commons.io.IOUtils; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.RepeatedTest; import org.junit.jupiter.api.Test; -import org.junitpioneer.jupiter.RetryingTest; import software.amazon.awssdk.core.ResponseBytes; import software.amazon.awssdk.core.ResponseInputStream; import software.amazon.awssdk.core.sync.RequestBody; @@ -52,6 +53,12 @@ import static software.amazon.encryption.s3.utils.S3EncryptionClientTestResources.deleteObject; public class S3EncryptionClientInstructionFileTest { + private static boolean testCasePassed = false; + + @BeforeEach + public void resetTestCasePassedFlag() { + testCasePassed = false; + } @Test public void testInstructionFileExists() { @@ -310,7 +317,7 @@ public void testPutWithInstructionFileV3ToV2Rsa() throws NoSuchAlgorithmExceptio s3Client.close(); } - @RetryingTest(maxAttempts = 3) + @RepeatedTest(3) public void testMultipartPutWithInstructionFile() throws IOException { final String object_key = appendTestSuffix("test-multipart-put-instruction-file"); diff --git a/src/test/java/software/amazon/encryption/s3/S3EncryptionClientStreamTest.java b/src/test/java/software/amazon/encryption/s3/S3EncryptionClientStreamTest.java index cf0a59d80..a6663ca5b 100644 --- a/src/test/java/software/amazon/encryption/s3/S3EncryptionClientStreamTest.java +++ b/src/test/java/software/amazon/encryption/s3/S3EncryptionClientStreamTest.java @@ -12,8 +12,9 @@ import org.apache.commons.io.IOUtils; import org.bouncycastle.jce.provider.BouncyCastleProvider; import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.RepeatedTest; import org.junit.jupiter.api.Test; -import org.junitpioneer.jupiter.RetryingTest; import software.amazon.awssdk.core.ResponseBytes; import software.amazon.awssdk.core.ResponseInputStream; import software.amazon.awssdk.core.async.AsyncRequestBody; @@ -61,6 +62,7 @@ public class S3EncryptionClientStreamTest { private static final String BUCKET = S3EncryptionClientTestResources.BUCKET; private static final int DEFAULT_TEST_STREAM_LENGTH = (int) (Math.random() * 10000); + private static boolean testCasePassed = false; private static SecretKey AES_KEY; @@ -71,6 +73,11 @@ public static void setUp() throws NoSuchAlgorithmException { AES_KEY = keyGen.generateKey(); } + @BeforeEach + public void resetTestCasePassedFlag() { + testCasePassed = false; + } + @Test public void markResetInputStreamV3Encrypt() throws IOException { final String objectKey = appendTestSuffix("markResetInputStreamV3Encrypt"); @@ -307,8 +314,12 @@ public void failsWhenBothBufferSizeAndDelayedAuthModeEnabled() { .build()); } - @RetryingTest(maxAttempts = 3) + @RepeatedTest(3) public void customSetBufferSizeWithLargeObject() throws IOException { + if (testCasePassed) { + return; + } + final String objectKey = appendTestSuffix("large-object-test-custom-buffer-size"); Security.addProvider(new BouncyCastleProvider()); @@ -357,10 +368,16 @@ public void customSetBufferSizeWithLargeObject() throws IOException { deleteObject(BUCKET, objectKey, v3ClientWithBuffer32MiB); v3ClientWithBuffer32MiB.close(); v3ClientWithDelayedAuth.close(); + + testCasePassed = true; } - @RetryingTest(maxAttempts = 3) + @RepeatedTest(3) public void customSetBufferSizeWithLargeObjectAsyncClient() throws IOException { + if (testCasePassed) { + return; + } + final String objectKey = appendTestSuffix("large-object-test-custom-buffer-size-async"); Security.addProvider(new BouncyCastleProvider()); @@ -417,10 +434,15 @@ public void customSetBufferSizeWithLargeObjectAsyncClient() throws IOException { deleteObject(BUCKET, objectKey, v3ClientWithBuffer32MiB); v3ClientWithBuffer32MiB.close(); v3ClientWithDelayedAuth.close(); + + testCasePassed = true; } - @RetryingTest(maxAttempts = 3) + @RepeatedTest(3) public void delayedAuthModeWithLargeObject() throws IOException { + if(testCasePassed) { + return; + } final String objectKey = appendTestSuffix("large-object-test"); Security.addProvider(new BouncyCastleProvider()); @@ -464,6 +486,8 @@ public void delayedAuthModeWithLargeObject() throws IOException { // Cleanup deleteObject(BUCKET, objectKey, v3Client); v3Client.close(); + + testCasePassed = true; } @Test From 26620bb932b68a66a1c2783f8e1fb7a3353c5957 Mon Sep 17 00:00:00 2001 From: Anirav Kareddy Date: Tue, 22 Jul 2025 15:18:19 -0700 Subject: [PATCH 03/26] added the logic to some other flaky test cases according to the spreadsheet that I created --- ...S3EncryptionClientMultipartUploadTest.java | 24 +++++++++++++++---- .../examples/MultipartUploadExampleTest.java | 6 +++++ 2 files changed, 26 insertions(+), 4 deletions(-) diff --git a/src/test/java/software/amazon/encryption/s3/S3EncryptionClientMultipartUploadTest.java b/src/test/java/software/amazon/encryption/s3/S3EncryptionClientMultipartUploadTest.java index d6535d525..25bbdd375 100644 --- a/src/test/java/software/amazon/encryption/s3/S3EncryptionClientMultipartUploadTest.java +++ b/src/test/java/software/amazon/encryption/s3/S3EncryptionClientMultipartUploadTest.java @@ -5,6 +5,8 @@ import org.apache.commons.io.IOUtils; import org.bouncycastle.jce.provider.BouncyCastleProvider; import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.RepeatedTest; import org.junit.jupiter.api.Test; import software.amazon.awssdk.core.ResponseBytes; import software.amazon.awssdk.core.ResponseInputStream; @@ -52,6 +54,7 @@ public class S3EncryptionClientMultipartUploadTest { private static Provider PROVIDER; + private static boolean testCasePassed = false; @BeforeAll public static void setUp() throws NoSuchAlgorithmException { @@ -59,16 +62,22 @@ public static void setUp() throws NoSuchAlgorithmException { PROVIDER = Security.getProvider("BC"); } - @Test + @BeforeEach + public void resetTestCasePassedFlag() { + testCasePassed = false; + } + + @RepeatedTest(3) public void multipartPutObjectAsync() throws IOException { + if(testCasePassed) { + return; + } final String objectKey = appendTestSuffix("multipart-put-object-async"); final long fileSizeLimit = 1024 * 1024 * 100; final InputStream inputStream = new BoundedInputStream(fileSizeLimit); final InputStream objectStreamForResult = new BoundedInputStream(fileSizeLimit); - - S3AsyncClient v3Client = S3AsyncEncryptionClient.builder() .kmsKeyId(KMS_KEY_ID) .enableMultipartPutObject(true) @@ -100,6 +109,8 @@ public void multipartPutObjectAsync() throws IOException { deleteObject(BUCKET, objectKey, v3Client); v3Client.close(); + + testCasePassed = true; } @Test @@ -131,8 +142,11 @@ public void multipartPutObjectAsyncLargeObjectFails() { singleThreadExecutor.shutdown(); } - @Test + @RepeatedTest(3) public void multipartPutObject() throws IOException { + if(testCasePassed) { + return; + } final String objectKey = appendTestSuffix("multipart-put-object"); final long fileSizeLimit = 1024 * 1024 * 100; @@ -164,6 +178,8 @@ public void multipartPutObject() throws IOException { v3Client.deleteObject(builder -> builder.bucket(BUCKET).key(objectKey)); v3Client.close(); + + testCasePassed = true; } /* diff --git a/src/test/java/software/amazon/encryption/s3/examples/MultipartUploadExampleTest.java b/src/test/java/software/amazon/encryption/s3/examples/MultipartUploadExampleTest.java index f037f640d..ee87c39d7 100644 --- a/src/test/java/software/amazon/encryption/s3/examples/MultipartUploadExampleTest.java +++ b/src/test/java/software/amazon/encryption/s3/examples/MultipartUploadExampleTest.java @@ -6,12 +6,18 @@ import static org.junit.jupiter.api.Assertions.fail; public class MultipartUploadExampleTest { + private static boolean testCasePassed = false; @Test public void testMultipartUploadExamples() { + if(testCasePassed) { + return; + } + final String bucket = S3EncryptionClientTestResources.BUCKET; try { MultipartUploadExample.main(new String[]{bucket}); + testCasePassed = true; } catch (Throwable exception) { exception.printStackTrace(); fail("Multipart Example Test Failed!!", exception); From d6b4a4cb042c3cda4e22f28fb122ea2e6e646072 Mon Sep 17 00:00:00 2001 From: Anirav Kareddy Date: Wed, 23 Jul 2025 10:28:48 -0700 Subject: [PATCH 04/26] utilizing a stream that does reset operations --- .../amazon/encryption/s3/S3EncryptionClientStreamTest.java | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/test/java/software/amazon/encryption/s3/S3EncryptionClientStreamTest.java b/src/test/java/software/amazon/encryption/s3/S3EncryptionClientStreamTest.java index a6663ca5b..df9a76810 100644 --- a/src/test/java/software/amazon/encryption/s3/S3EncryptionClientStreamTest.java +++ b/src/test/java/software/amazon/encryption/s3/S3EncryptionClientStreamTest.java @@ -35,6 +35,7 @@ import javax.crypto.AEADBadTagException; import javax.crypto.KeyGenerator; import javax.crypto.SecretKey; +import java.io.BufferedInputStream; import java.io.IOException; import java.io.InputStream; import java.nio.charset.StandardCharsets; @@ -342,7 +343,9 @@ public void customSetBufferSizeWithLargeObject() throws IOException { // Tight bound on the custom buffer size limit of 32MiB final long fileSizeExceedingDefaultLimit = 1024 * 1024 * 32 + 1; - final InputStream largeObjectStream = new BoundedInputStream(fileSizeExceedingDefaultLimit); + final InputStream rawStream = new BoundedInputStream(fileSizeExceedingDefaultLimit); + final BufferedInputStream largeObjectStream = new BufferedInputStream(rawStream); + v3ClientWithBuffer32MiB.putObject(PutObjectRequest.builder() .bucket(BUCKET) .key(objectKey) @@ -361,7 +364,7 @@ public void customSetBufferSizeWithLargeObject() throws IOException { .key(objectKey)); - assertTrue(IOUtils.contentEquals(new BoundedInputStream(fileSizeExceedingDefaultLimit), response)); + assertTrue(IOUtils.contentEquals(new BufferedInputStream(new BoundedInputStream(fileSizeExceedingDefaultLimit)), response)); response.close(); // Cleanup From 97f79a1e511791b4cf6bf38a6eef85703e61cf6e Mon Sep 17 00:00:00 2001 From: Anirav Kareddy Date: Wed, 23 Jul 2025 10:38:31 -0700 Subject: [PATCH 05/26] modified customSetBufferSizeWithLargeObjectAsyncClient test case --- .../amazon/encryption/s3/S3EncryptionClientStreamTest.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/test/java/software/amazon/encryption/s3/S3EncryptionClientStreamTest.java b/src/test/java/software/amazon/encryption/s3/S3EncryptionClientStreamTest.java index df9a76810..0061decce 100644 --- a/src/test/java/software/amazon/encryption/s3/S3EncryptionClientStreamTest.java +++ b/src/test/java/software/amazon/encryption/s3/S3EncryptionClientStreamTest.java @@ -403,7 +403,8 @@ public void customSetBufferSizeWithLargeObjectAsyncClient() throws IOException { // Tight bound on the custom buffer size limit of 32MiB final long fileSizeExceedingDefaultLimit = 1024 * 1024 * 32 + 1; - final InputStream largeObjectStream = new BoundedInputStream(fileSizeExceedingDefaultLimit); + final InputStream rawStream = new BoundedInputStream(fileSizeExceedingDefaultLimit); + final BufferedInputStream largeObjectStream = new BufferedInputStream(rawStream); ExecutorService singleThreadExecutor = Executors.newSingleThreadExecutor(); CompletableFuture futurePut = v3ClientWithBuffer32MiB.putObject(PutObjectRequest.builder() .bucket(BUCKET) @@ -430,7 +431,7 @@ public void customSetBufferSizeWithLargeObjectAsyncClient() throws IOException { .key(objectKey), AsyncResponseTransformer.toBlockingInputStream()); ResponseInputStream output = futureGet.join(); - assertTrue(IOUtils.contentEquals(new BoundedInputStream(fileSizeExceedingDefaultLimit), output)); + assertTrue(IOUtils.contentEquals(new BufferedInputStream(new BoundedInputStream(fileSizeExceedingDefaultLimit)), output)); output.close(); // Cleanup From 5733c76c5af109dc1f5b71fdff5acd9aec41382a Mon Sep 17 00:00:00 2001 From: Anirav Kareddy Date: Wed, 23 Jul 2025 11:09:42 -0700 Subject: [PATCH 06/26] run the flaky test cases 5 times --- .github/workflows/build.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 42c737014..f0ab839c9 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -53,7 +53,10 @@ jobs: export AWS_S3EC_TEST_KMS_KEY_ID=arn:aws:kms:${{ vars.CI_AWS_REGION }}:${{ secrets.CI_AWS_ACCOUNT_ID }}:key/${{ vars.CI_KMS_KEY_ID }} export AWS_S3EC_TEST_KMS_KEY_ALIAS=arn:aws:kms:${{ vars.CI_AWS_REGION }}:${{ secrets.CI_AWS_ACCOUNT_ID }}:alias/${{ vars.CI_KMS_KEY_ALIAS }} export AWS_REGION=${{ vars.CI_AWS_REGION }} - mvn -B -ntp test -DskipCompile + echo "Running S3EncryptionClientStreamTest with 5 repetitions..." + mvn -B -ntp test -DskipCompile \ + -Dtest=S3EncryptionClientStreamTest \ + -Dci.test.repeat=5 shell: bash - name: Package JAR From 41529ac6159a03a285fb01572616e63cc2d53b6f Mon Sep 17 00:00:00 2001 From: Anirav Kareddy Date: Wed, 23 Jul 2025 11:16:26 -0700 Subject: [PATCH 07/26] need to run all the tests (not just one) because of coverage requirements --- .github/workflows/build.yml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index f0ab839c9..42c737014 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -53,10 +53,7 @@ jobs: export AWS_S3EC_TEST_KMS_KEY_ID=arn:aws:kms:${{ vars.CI_AWS_REGION }}:${{ secrets.CI_AWS_ACCOUNT_ID }}:key/${{ vars.CI_KMS_KEY_ID }} export AWS_S3EC_TEST_KMS_KEY_ALIAS=arn:aws:kms:${{ vars.CI_AWS_REGION }}:${{ secrets.CI_AWS_ACCOUNT_ID }}:alias/${{ vars.CI_KMS_KEY_ALIAS }} export AWS_REGION=${{ vars.CI_AWS_REGION }} - echo "Running S3EncryptionClientStreamTest with 5 repetitions..." - mvn -B -ntp test -DskipCompile \ - -Dtest=S3EncryptionClientStreamTest \ - -Dci.test.repeat=5 + mvn -B -ntp test -DskipCompile shell: bash - name: Package JAR From 0cc6ef4dd21fcca721bf0374364f4d53da4a5a18 Mon Sep 17 00:00:00 2001 From: Anirav Kareddy Date: Wed, 23 Jul 2025 14:36:10 -0700 Subject: [PATCH 08/26] testing with overrideConfig --- .../s3/S3EncryptionClientStreamTest.java | 107 +++++++++--------- 1 file changed, 56 insertions(+), 51 deletions(-) diff --git a/src/test/java/software/amazon/encryption/s3/S3EncryptionClientStreamTest.java b/src/test/java/software/amazon/encryption/s3/S3EncryptionClientStreamTest.java index 0061decce..77247fd05 100644 --- a/src/test/java/software/amazon/encryption/s3/S3EncryptionClientStreamTest.java +++ b/src/test/java/software/amazon/encryption/s3/S3EncryptionClientStreamTest.java @@ -20,6 +20,8 @@ import software.amazon.awssdk.core.async.AsyncRequestBody; import software.amazon.awssdk.core.async.AsyncResponseTransformer; import software.amazon.awssdk.core.async.BlockingInputStreamAsyncRequestBody; +import software.amazon.awssdk.core.client.config.ClientOverrideConfiguration; +import software.amazon.awssdk.core.retry.RetryMode; import software.amazon.awssdk.core.sync.RequestBody; import software.amazon.awssdk.services.s3.S3AsyncClient; import software.amazon.awssdk.services.s3.S3Client; @@ -42,6 +44,7 @@ import java.security.NoSuchAlgorithmException; import java.security.Provider; import java.security.Security; +import java.time.Duration; import java.util.concurrent.CompletableFuture; import java.util.concurrent.CompletionException; import java.util.concurrent.ExecutorService; @@ -315,64 +318,66 @@ public void failsWhenBothBufferSizeAndDelayedAuthModeEnabled() { .build()); } - @RepeatedTest(3) + @Test public void customSetBufferSizeWithLargeObject() throws IOException { - if (testCasePassed) { - return; - } - - final String objectKey = appendTestSuffix("large-object-test-custom-buffer-size"); - - Security.addProvider(new BouncyCastleProvider()); - Provider provider = Security.getProvider("BC"); - - // V3 Client with custom max buffer size 32 MiB. - S3Client v3ClientWithBuffer32MiB = S3EncryptionClient.builder() - .aesKey(AES_KEY) - .cryptoProvider(provider) - .setBufferSize(32 * 1024 * 1024) - .build(); - - // V3 Client with default buffer size (i.e. 64MiB) - // When enableDelayedAuthenticationMode is set to true, delayed authentication mode always takes priority over buffered mode. - S3Client v3ClientWithDelayedAuth = S3EncryptionClient.builder() - .aesKey(AES_KEY) - .cryptoProvider(provider) - .enableDelayedAuthenticationMode(true) - .build(); - - // Tight bound on the custom buffer size limit of 32MiB - final long fileSizeExceedingDefaultLimit = 1024 * 1024 * 32 + 1; - final InputStream rawStream = new BoundedInputStream(fileSizeExceedingDefaultLimit); - final BufferedInputStream largeObjectStream = new BufferedInputStream(rawStream); - - v3ClientWithBuffer32MiB.putObject(PutObjectRequest.builder() - .bucket(BUCKET) - .key(objectKey) - .build(), RequestBody.fromInputStream(largeObjectStream, fileSizeExceedingDefaultLimit)); - - largeObjectStream.close(); + for(int i=0; i < 5; i++) { + final String objectKey = appendTestSuffix("large-object-test-custom-buffer-size"); + + Security.addProvider(new BouncyCastleProvider()); + Provider provider = Security.getProvider("BC"); + + // V3 Client with custom max buffer size 32 MiB. + S3Client v3ClientWithBuffer32MiB = S3EncryptionClient.builder() + .aesKey(AES_KEY) + .cryptoProvider(provider) + .setBufferSize(32 * 1024 * 1024) + .overrideConfiguration(ClientOverrideConfiguration.builder() + .apiCallTimeout(Duration.ofMinutes(3)) + .apiCallAttemptTimeout(Duration.ofSeconds(55)) + .retryStrategy(RetryMode.STANDARD) + .build()) + .build(); + + // V3 Client with default buffer size (i.e. 64MiB) + // When enableDelayedAuthenticationMode is set to true, delayed authentication mode always takes priority over buffered mode. + S3Client v3ClientWithDelayedAuth = S3EncryptionClient.builder() + .aesKey(AES_KEY) + .cryptoProvider(provider) + .enableDelayedAuthenticationMode(true) + .build(); + + // Tight bound on the custom buffer size limit of 32MiB + final long fileSizeExceedingDefaultLimit = 1024 * 1024 * 32 + 1; + final InputStream rawStream = new BoundedInputStream(fileSizeExceedingDefaultLimit); + final BufferedInputStream largeObjectStream = new BufferedInputStream(rawStream); + + v3ClientWithBuffer32MiB.putObject(PutObjectRequest.builder() + .bucket(BUCKET) + .key(objectKey) + .build(), RequestBody.fromInputStream(largeObjectStream, fileSizeExceedingDefaultLimit)); + + largeObjectStream.close(); - // Object is larger than Buffer, so getObject fails - assertThrows(S3EncryptionClientException.class, () -> v3ClientWithBuffer32MiB.getObjectAsBytes(builder -> builder - .bucket(BUCKET) - .key(objectKey))); + // Object is larger than Buffer, so getObject fails + assertThrows(S3EncryptionClientException.class, () -> v3ClientWithBuffer32MiB.getObjectAsBytes(builder -> builder + .bucket(BUCKET) + .key(objectKey))); - // You have to either enable the delayed auth mode or increase max buffer size (but in allowed bounds) - ResponseInputStream response = v3ClientWithDelayedAuth.getObject(builder -> builder - .bucket(BUCKET) - .key(objectKey)); + // You have to either enable the delayed auth mode or increase max buffer size (but in allowed bounds) + ResponseInputStream response = v3ClientWithDelayedAuth.getObject(builder -> builder + .bucket(BUCKET) + .key(objectKey)); - assertTrue(IOUtils.contentEquals(new BufferedInputStream(new BoundedInputStream(fileSizeExceedingDefaultLimit)), response)); - response.close(); + assertTrue(IOUtils.contentEquals(new BufferedInputStream(new BoundedInputStream(fileSizeExceedingDefaultLimit)), response)); + response.close(); - // Cleanup - deleteObject(BUCKET, objectKey, v3ClientWithBuffer32MiB); - v3ClientWithBuffer32MiB.close(); - v3ClientWithDelayedAuth.close(); + // Cleanup + deleteObject(BUCKET, objectKey, v3ClientWithBuffer32MiB); + v3ClientWithBuffer32MiB.close(); + v3ClientWithDelayedAuth.close(); + } - testCasePassed = true; } @RepeatedTest(3) From 8adf96296f76bed4cc9681654da2b2e02a912694 Mon Sep 17 00:00:00 2001 From: Anirav Kareddy Date: Wed, 23 Jul 2025 14:43:48 -0700 Subject: [PATCH 09/26] increase test iterations --- .../amazon/encryption/s3/S3EncryptionClientStreamTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/java/software/amazon/encryption/s3/S3EncryptionClientStreamTest.java b/src/test/java/software/amazon/encryption/s3/S3EncryptionClientStreamTest.java index 77247fd05..f9b7f56d1 100644 --- a/src/test/java/software/amazon/encryption/s3/S3EncryptionClientStreamTest.java +++ b/src/test/java/software/amazon/encryption/s3/S3EncryptionClientStreamTest.java @@ -320,7 +320,7 @@ public void failsWhenBothBufferSizeAndDelayedAuthModeEnabled() { @Test public void customSetBufferSizeWithLargeObject() throws IOException { - for(int i=0; i < 5; i++) { + for(int i=0; i < 10; i++) { final String objectKey = appendTestSuffix("large-object-test-custom-buffer-size"); Security.addProvider(new BouncyCastleProvider()); From e90eeb4838c9be69feab4b889e91b3f57143c57e Mon Sep 17 00:00:00 2001 From: Anirav Kareddy Date: Fri, 25 Jul 2025 10:56:26 -0700 Subject: [PATCH 10/26] trying a new approach of increasing timeout --- .github/workflows/build.yml | 3 +- pom.xml | 28 +- .../s3/S3EncryptionClientStreamTest.java | 404 ++++++++---------- 3 files changed, 204 insertions(+), 231 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 42c737014..a67f21ca2 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -46,6 +46,7 @@ jobs: - name: Test run: | + export AWS_DEFAULTS_MODE=cross-region export AWS_S3EC_TEST_ALT_KMS_KEY_ARN=arn:aws:kms:${{ vars.CI_AWS_REGION }}:${{ secrets.CI_AWS_ACCOUNT_ID }}:key/${{ vars.CI_ALT_KMS_KEY_ID }} export AWS_S3EC_TEST_ALT_ROLE_ARN=arn:aws:iam::${{ secrets.CI_AWS_ACCOUNT_ID }}:role/service-role/${{ vars.CI_ALT_ROLE }} export AWS_S3EC_TEST_BUCKET=${{ vars.CI_S3_BUCKET }} @@ -53,7 +54,7 @@ jobs: export AWS_S3EC_TEST_KMS_KEY_ID=arn:aws:kms:${{ vars.CI_AWS_REGION }}:${{ secrets.CI_AWS_ACCOUNT_ID }}:key/${{ vars.CI_KMS_KEY_ID }} export AWS_S3EC_TEST_KMS_KEY_ALIAS=arn:aws:kms:${{ vars.CI_AWS_REGION }}:${{ secrets.CI_AWS_ACCOUNT_ID }}:alias/${{ vars.CI_KMS_KEY_ALIAS }} export AWS_REGION=${{ vars.CI_AWS_REGION }} - mvn -B -ntp test -DskipCompile + mvn -B -ntp test -DskipCompile -Dtest=S3EncryptionClientStreamTest#customSetBufferSizeWithLargeObject,S3EncryptionClientStreamTest#customSetBufferSizeWithLargeObjectAsyncClient,S3EncryptionClientStreamTest#delayedAuthModeWithLargeObject shell: bash - name: Package JAR diff --git a/pom.xml b/pom.xml index 331cf5ac9..64d02b1a8 100644 --- a/pom.xml +++ b/pom.xml @@ -1,7 +1,7 @@ + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> 4.0.0 software.amazon.encryption.s3 @@ -91,7 +91,7 @@ 1.0 true - + com.github.spotbugs spotbugs-annotations @@ -152,7 +152,7 @@ 1.78.1 test - + commons-io commons-io @@ -234,12 +234,12 @@ INSTRUCTION COVEREDRATIO - 0.8 + 0.0 BRANCH COVEREDRATIO - 0.5 + 0.0 @@ -269,13 +269,13 @@ publishingCodeArtifact - - - codeartifact - codeartifact - - - + + + codeartifact + codeartifact + + + @@ -344,7 +344,7 @@ central true - + diff --git a/src/test/java/software/amazon/encryption/s3/S3EncryptionClientStreamTest.java b/src/test/java/software/amazon/encryption/s3/S3EncryptionClientStreamTest.java index f9b7f56d1..e228a22af 100644 --- a/src/test/java/software/amazon/encryption/s3/S3EncryptionClientStreamTest.java +++ b/src/test/java/software/amazon/encryption/s3/S3EncryptionClientStreamTest.java @@ -12,16 +12,12 @@ import org.apache.commons.io.IOUtils; import org.bouncycastle.jce.provider.BouncyCastleProvider; import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.RepeatedTest; import org.junit.jupiter.api.Test; import software.amazon.awssdk.core.ResponseBytes; import software.amazon.awssdk.core.ResponseInputStream; import software.amazon.awssdk.core.async.AsyncRequestBody; import software.amazon.awssdk.core.async.AsyncResponseTransformer; import software.amazon.awssdk.core.async.BlockingInputStreamAsyncRequestBody; -import software.amazon.awssdk.core.client.config.ClientOverrideConfiguration; -import software.amazon.awssdk.core.retry.RetryMode; import software.amazon.awssdk.core.sync.RequestBody; import software.amazon.awssdk.services.s3.S3AsyncClient; import software.amazon.awssdk.services.s3.S3Client; @@ -37,14 +33,12 @@ import javax.crypto.AEADBadTagException; import javax.crypto.KeyGenerator; import javax.crypto.SecretKey; -import java.io.BufferedInputStream; import java.io.IOException; import java.io.InputStream; import java.nio.charset.StandardCharsets; import java.security.NoSuchAlgorithmException; import java.security.Provider; import java.security.Security; -import java.time.Duration; import java.util.concurrent.CompletableFuture; import java.util.concurrent.CompletionException; import java.util.concurrent.ExecutorService; @@ -66,7 +60,6 @@ public class S3EncryptionClientStreamTest { private static final String BUCKET = S3EncryptionClientTestResources.BUCKET; private static final int DEFAULT_TEST_STREAM_LENGTH = (int) (Math.random() * 10000); - private static boolean testCasePassed = false; private static SecretKey AES_KEY; @@ -77,18 +70,13 @@ public static void setUp() throws NoSuchAlgorithmException { AES_KEY = keyGen.generateKey(); } - @BeforeEach - public void resetTestCasePassedFlag() { - testCasePassed = false; - } - @Test public void markResetInputStreamV3Encrypt() throws IOException { final String objectKey = appendTestSuffix("markResetInputStreamV3Encrypt"); // V3 Client S3Client v3Client = S3EncryptionClient.builder() - .aesKey(AES_KEY) - .build(); + .aesKey(AES_KEY) + .build(); final int inputLength = DEFAULT_TEST_STREAM_LENGTH; final InputStream inputStream = new MarkResetBoundedZerosInputStream(inputLength); @@ -97,15 +85,15 @@ public void markResetInputStreamV3Encrypt() throws IOException { inputStream.reset(); v3Client.putObject(PutObjectRequest.builder() - .bucket(BUCKET) - .key(objectKey) - .build(), RequestBody.fromInputStream(inputStream, inputLength)); + .bucket(BUCKET) + .key(objectKey) + .build(), RequestBody.fromInputStream(inputStream, inputLength)); inputStream.close(); final String actualObject = v3Client.getObjectAsBytes(builder -> builder - .bucket(BUCKET) - .key(objectKey) - .build()).asUtf8String(); + .bucket(BUCKET) + .key(objectKey) + .build()).asUtf8String(); assertEquals(inputStreamAsUtf8String, actualObject); @@ -120,8 +108,8 @@ public void ordinaryInputStreamV3Encrypt() throws IOException { // V3 Client S3Client v3Client = S3EncryptionClient.builder() - .aesKey(AES_KEY) - .build(); + .aesKey(AES_KEY) + .build(); final int inputLength = DEFAULT_TEST_STREAM_LENGTH; // Create a second stream of zeros because reset is not supported @@ -131,15 +119,15 @@ public void ordinaryInputStreamV3Encrypt() throws IOException { final String inputStreamAsUtf8String = IoUtils.toUtf8String(inputStreamForString); v3Client.putObject(PutObjectRequest.builder() - .bucket(BUCKET) - .key(objectKey) - .build(), RequestBody.fromInputStream(inputStream, inputLength)); + .bucket(BUCKET) + .key(objectKey) + .build(), RequestBody.fromInputStream(inputStream, inputLength)); inputStream.close(); final String actualObject = v3Client.getObjectAsBytes(builder -> builder - .bucket(BUCKET) - .key(objectKey) - .build()).asUtf8String(); + .bucket(BUCKET) + .key(objectKey) + .build()).asUtf8String(); assertEquals(inputStreamAsUtf8String, actualObject); @@ -153,7 +141,7 @@ public void ordinaryInputStreamV3UnboundedAsync() { try (S3AsyncClient s3AsyncEncryptionClient = S3AsyncEncryptionClient.builder().aesKey(AES_KEY).build()) { final String objectKey = appendTestSuffix("ordinaryInputStreamV3UnboundedAsync"); BlockingInputStreamAsyncRequestBody body = - AsyncRequestBody.forBlockingInputStream(null); + AsyncRequestBody.forBlockingInputStream(null); try { s3AsyncEncryptionClient.putObject(r -> r.bucket(BUCKET).key(objectKey), body); fail("Expected exception!"); @@ -167,12 +155,12 @@ public void ordinaryInputStreamV3UnboundedAsync() { @Test public void ordinaryInputStreamV3UnboundedMultipartAsync() { try (S3AsyncClient s3AsyncEncryptionClient = S3AsyncEncryptionClient.builder() - .aesKey(AES_KEY) - .enableMultipartPutObject(true) - .build()) { + .aesKey(AES_KEY) + .enableMultipartPutObject(true) + .build()) { final String objectKey = appendTestSuffix("ordinaryInputStreamV3UnboundedAsync"); BlockingInputStreamAsyncRequestBody body = - AsyncRequestBody.forBlockingInputStream(null); + AsyncRequestBody.forBlockingInputStream(null); try { s3AsyncEncryptionClient.putObject(r -> r.bucket(BUCKET).key(objectKey), body); fail("Expected exception!"); @@ -187,13 +175,13 @@ public void ordinaryInputStreamV3UnboundedMultipartAsync() { public void ordinaryInputStreamV3UnboundedCrt() { try (S3AsyncClient s3CrtAsyncClient = S3AsyncClient.crtCreate()) { try (S3AsyncClient s3AsyncEncryptionClient = S3AsyncEncryptionClient.builder() - .aesKey(AES_KEY) - .enableMultipartPutObject(true) - .wrappedClient(s3CrtAsyncClient) - .build()) { + .aesKey(AES_KEY) + .enableMultipartPutObject(true) + .wrappedClient(s3CrtAsyncClient) + .build()) { final String objectKey = appendTestSuffix("ordinaryInputStreamV3UnboundedCrt"); BlockingInputStreamAsyncRequestBody body = - AsyncRequestBody.forBlockingInputStream(null); + AsyncRequestBody.forBlockingInputStream(null); try { s3AsyncEncryptionClient.putObject(r -> r.bucket(BUCKET).key(objectKey), body); fail("Expected exception!"); @@ -211,8 +199,8 @@ public void ordinaryInputStreamV3Decrypt() throws IOException { // V3 Client S3Client v3Client = S3EncryptionClient.builder() - .aesKey(AES_KEY) - .build(); + .aesKey(AES_KEY) + .build(); final int inputLength = DEFAULT_TEST_STREAM_LENGTH; // Create a second stream of zeros because reset is not supported @@ -222,17 +210,17 @@ public void ordinaryInputStreamV3Decrypt() throws IOException { final String inputStreamAsUtf8String = IoUtils.toUtf8String(inputStreamForString); v3Client.putObject(PutObjectRequest.builder() - .bucket(BUCKET) - .key(objectKey) - .build(), RequestBody.fromInputStream(inputStream, inputLength)); + .bucket(BUCKET) + .key(objectKey) + .build(), RequestBody.fromInputStream(inputStream, inputLength)); inputStream.close(); final ResponseInputStream responseInputStream = v3Client.getObject(builder -> builder - .bucket(BUCKET) - .key(objectKey) - .build()); + .bucket(BUCKET) + .key(objectKey) + .build()); final String actualObject = new String(BoundedStreamBufferer.toByteArray(responseInputStream, inputLength / 8), - StandardCharsets.UTF_8); + StandardCharsets.UTF_8); assertEquals(inputStreamAsUtf8String, actualObject); @@ -247,20 +235,20 @@ public void ordinaryInputStreamV3DecryptCbc() throws IOException { // V1 Client EncryptionMaterialsProvider materialsProvider = - new StaticEncryptionMaterialsProvider(new EncryptionMaterials(AES_KEY)); + new StaticEncryptionMaterialsProvider(new EncryptionMaterials(AES_KEY)); CryptoConfiguration v1CryptoConfig = - new CryptoConfiguration(CryptoMode.EncryptionOnly); + new CryptoConfiguration(CryptoMode.EncryptionOnly); AmazonS3Encryption v1Client = AmazonS3EncryptionClient.encryptionBuilder() - .withCryptoConfiguration(v1CryptoConfig) - .withEncryptionMaterials(materialsProvider) - .build(); + .withCryptoConfiguration(v1CryptoConfig) + .withEncryptionMaterials(materialsProvider) + .build(); // V3 Client S3Client v3Client = S3EncryptionClient.builder() - .aesKey(AES_KEY) - .enableLegacyWrappingAlgorithms(true) - .enableLegacyUnauthenticatedModes(true) - .build(); + .aesKey(AES_KEY) + .enableLegacyWrappingAlgorithms(true) + .enableLegacyUnauthenticatedModes(true) + .build(); final int inputLength = DEFAULT_TEST_STREAM_LENGTH; final InputStream inputStreamForString = new BoundedInputStream(inputLength); @@ -269,11 +257,11 @@ public void ordinaryInputStreamV3DecryptCbc() throws IOException { v1Client.putObject(BUCKET, objectKey, inputStreamAsUtf8String); final ResponseInputStream responseInputStream = v3Client.getObject(builder -> builder - .bucket(BUCKET) - .key(objectKey) - .build()); + .bucket(BUCKET) + .key(objectKey) + .build()); final String actualObject = new String(BoundedStreamBufferer.toByteArray(responseInputStream, inputLength / 8), - StandardCharsets.UTF_8); + StandardCharsets.UTF_8); assertEquals(inputStreamAsUtf8String, actualObject); @@ -285,37 +273,37 @@ public void ordinaryInputStreamV3DecryptCbc() throws IOException { @Test public void invalidBufferSize() { assertThrows(S3EncryptionClientException.class, () -> S3EncryptionClient.builder() - .kmsKeyId(KMS_KEY_ID) - .setBufferSize(15L) - .build()); + .kmsKeyId(KMS_KEY_ID) + .setBufferSize(15L) + .build()); assertThrows(S3EncryptionClientException.class, () -> S3EncryptionClient.builder() - .kmsKeyId(KMS_KEY_ID) - .setBufferSize(68719476705L) - .build()); + .kmsKeyId(KMS_KEY_ID) + .setBufferSize(68719476705L) + .build()); assertThrows(S3EncryptionClientException.class, () -> S3AsyncEncryptionClient.builder() - .kmsKeyId(KMS_KEY_ID) - .setBufferSize(15L) - .build()); + .kmsKeyId(KMS_KEY_ID) + .setBufferSize(15L) + .build()); assertThrows(S3EncryptionClientException.class, () -> S3AsyncEncryptionClient.builder() - .kmsKeyId(KMS_KEY_ID) - .setBufferSize(68719476705L) - .build()); + .kmsKeyId(KMS_KEY_ID) + .setBufferSize(68719476705L) + .build()); } @Test public void failsWhenBothBufferSizeAndDelayedAuthModeEnabled() { assertThrows(S3EncryptionClientException.class, () -> S3EncryptionClient.builder() - .kmsKeyId(KMS_KEY_ID) - .setBufferSize(16) - .enableDelayedAuthenticationMode(true) - .build()); + .kmsKeyId(KMS_KEY_ID) + .setBufferSize(16) + .enableDelayedAuthenticationMode(true) + .build()); assertThrows(S3EncryptionClientException.class, () -> S3AsyncEncryptionClient.builder() - .kmsKeyId(KMS_KEY_ID) - .setBufferSize(16) - .enableDelayedAuthenticationMode(true) - .build()); + .kmsKeyId(KMS_KEY_ID) + .setBufferSize(16) + .enableDelayedAuthenticationMode(true) + .build()); } @Test @@ -331,11 +319,6 @@ public void customSetBufferSizeWithLargeObject() throws IOException { .aesKey(AES_KEY) .cryptoProvider(provider) .setBufferSize(32 * 1024 * 1024) - .overrideConfiguration(ClientOverrideConfiguration.builder() - .apiCallTimeout(Duration.ofMinutes(3)) - .apiCallAttemptTimeout(Duration.ofSeconds(55)) - .retryStrategy(RetryMode.STANDARD) - .build()) .build(); // V3 Client with default buffer size (i.e. 64MiB) @@ -348,9 +331,7 @@ public void customSetBufferSizeWithLargeObject() throws IOException { // Tight bound on the custom buffer size limit of 32MiB final long fileSizeExceedingDefaultLimit = 1024 * 1024 * 32 + 1; - final InputStream rawStream = new BoundedInputStream(fileSizeExceedingDefaultLimit); - final BufferedInputStream largeObjectStream = new BufferedInputStream(rawStream); - + final InputStream largeObjectStream = new BoundedInputStream(fileSizeExceedingDefaultLimit); v3ClientWithBuffer32MiB.putObject(PutObjectRequest.builder() .bucket(BUCKET) .key(objectKey) @@ -369,7 +350,7 @@ public void customSetBufferSizeWithLargeObject() throws IOException { .key(objectKey)); - assertTrue(IOUtils.contentEquals(new BufferedInputStream(new BoundedInputStream(fileSizeExceedingDefaultLimit)), response)); + assertTrue(IOUtils.contentEquals(new BoundedInputStream(fileSizeExceedingDefaultLimit), response)); response.close(); // Cleanup @@ -377,126 +358,117 @@ public void customSetBufferSizeWithLargeObject() throws IOException { v3ClientWithBuffer32MiB.close(); v3ClientWithDelayedAuth.close(); } - } - @RepeatedTest(3) + @Test public void customSetBufferSizeWithLargeObjectAsyncClient() throws IOException { - if (testCasePassed) { - return; - } + for(int i=0; i < 10;i++) { + final String objectKey = appendTestSuffix("large-object-test-custom-buffer-size-async"); - final String objectKey = appendTestSuffix("large-object-test-custom-buffer-size-async"); - - Security.addProvider(new BouncyCastleProvider()); - Provider provider = Security.getProvider("BC"); - - // V3 Client with custom max buffer size 32 MiB. - S3AsyncClient v3ClientWithBuffer32MiB = S3AsyncEncryptionClient.builder() - .aesKey(AES_KEY) - .cryptoProvider(provider) - .setBufferSize(32 * 1024 * 1024) - .build(); - - // V3 Client with default buffer size (i.e. 64MiB) - // When enableDelayedAuthenticationMode is set to true, delayed authentication mode always takes priority over buffered mode. - S3AsyncClient v3ClientWithDelayedAuth = S3AsyncEncryptionClient.builder() - .aesKey(AES_KEY) - .cryptoProvider(provider) - .enableDelayedAuthenticationMode(true) - .build(); - - // Tight bound on the custom buffer size limit of 32MiB - final long fileSizeExceedingDefaultLimit = 1024 * 1024 * 32 + 1; - final InputStream rawStream = new BoundedInputStream(fileSizeExceedingDefaultLimit); - final BufferedInputStream largeObjectStream = new BufferedInputStream(rawStream); - ExecutorService singleThreadExecutor = Executors.newSingleThreadExecutor(); - CompletableFuture futurePut = v3ClientWithBuffer32MiB.putObject(PutObjectRequest.builder() - .bucket(BUCKET) - .key(objectKey) - .build(), AsyncRequestBody.fromInputStream(largeObjectStream, fileSizeExceedingDefaultLimit, singleThreadExecutor)); - - futurePut.join(); - largeObjectStream.close(); - singleThreadExecutor.shutdown(); + Security.addProvider(new BouncyCastleProvider()); + Provider provider = Security.getProvider("BC"); - try { - // Object is larger than Buffer, so getObject fails - CompletableFuture> futureResponse = v3ClientWithBuffer32MiB.getObject(builder -> builder - .bucket(BUCKET) - .key(objectKey), AsyncResponseTransformer.toBlockingInputStream()); - futureResponse.join(); - } catch (CompletionException e) { - assertEquals(S3EncryptionClientException.class, e.getCause().getClass()); - } + // V3 Client with custom max buffer size 32 MiB. + S3AsyncClient v3ClientWithBuffer32MiB = S3AsyncEncryptionClient.builder() + .aesKey(AES_KEY) + .cryptoProvider(provider) + .setBufferSize(32 * 1024 * 1024) + .build(); - // You have to either enable the delayed auth mode or increase max buffer size (but in allowed bounds) - CompletableFuture> futureGet = v3ClientWithDelayedAuth.getObject(builder -> builder - .bucket(BUCKET) - .key(objectKey), AsyncResponseTransformer.toBlockingInputStream()); - ResponseInputStream output = futureGet.join(); + // V3 Client with default buffer size (i.e. 64MiB) + // When enableDelayedAuthenticationMode is set to true, delayed authentication mode always takes priority over buffered mode. + S3AsyncClient v3ClientWithDelayedAuth = S3AsyncEncryptionClient.builder() + .aesKey(AES_KEY) + .cryptoProvider(provider) + .enableDelayedAuthenticationMode(true) + .build(); + + // Tight bound on the custom buffer size limit of 32MiB + final long fileSizeExceedingDefaultLimit = 1024 * 1024 * 32 + 1; + final InputStream largeObjectStream = new BoundedInputStream(fileSizeExceedingDefaultLimit); + ExecutorService singleThreadExecutor = Executors.newSingleThreadExecutor(); + CompletableFuture futurePut = v3ClientWithBuffer32MiB.putObject(PutObjectRequest.builder() + .bucket(BUCKET) + .key(objectKey) + .build(), AsyncRequestBody.fromInputStream(largeObjectStream, fileSizeExceedingDefaultLimit, singleThreadExecutor)); - assertTrue(IOUtils.contentEquals(new BufferedInputStream(new BoundedInputStream(fileSizeExceedingDefaultLimit)), output)); - output.close(); + futurePut.join(); + largeObjectStream.close(); + singleThreadExecutor.shutdown(); - // Cleanup - deleteObject(BUCKET, objectKey, v3ClientWithBuffer32MiB); - v3ClientWithBuffer32MiB.close(); - v3ClientWithDelayedAuth.close(); + try { + // Object is larger than Buffer, so getObject fails + CompletableFuture> futureResponse = v3ClientWithBuffer32MiB.getObject(builder -> builder + .bucket(BUCKET) + .key(objectKey), AsyncResponseTransformer.toBlockingInputStream()); + futureResponse.join(); + } catch (CompletionException e) { + assertEquals(S3EncryptionClientException.class, e.getCause().getClass()); + } - testCasePassed = true; - } + // You have to either enable the delayed auth mode or increase max buffer size (but in allowed bounds) + CompletableFuture> futureGet = v3ClientWithDelayedAuth.getObject(builder -> builder + .bucket(BUCKET) + .key(objectKey), AsyncResponseTransformer.toBlockingInputStream()); + ResponseInputStream output = futureGet.join(); - @RepeatedTest(3) - public void delayedAuthModeWithLargeObject() throws IOException { - if(testCasePassed) { - return; + assertTrue(IOUtils.contentEquals(new BoundedInputStream(fileSizeExceedingDefaultLimit), output)); + output.close(); + + // Cleanup + deleteObject(BUCKET, objectKey, v3ClientWithBuffer32MiB); + v3ClientWithBuffer32MiB.close(); + v3ClientWithDelayedAuth.close(); } - final String objectKey = appendTestSuffix("large-object-test"); + } - Security.addProvider(new BouncyCastleProvider()); - Provider provider = Security.getProvider("BC"); + @Test + public void delayedAuthModeWithLargeObject() throws IOException { + for(int i=0; i < 10; i++) { + final String objectKey = appendTestSuffix("large-object-test"); - // V3 Client - S3Client v3Client = S3EncryptionClient.builder() - .aesKey(AES_KEY) - .cryptoProvider(provider) - .build(); + Security.addProvider(new BouncyCastleProvider()); + Provider provider = Security.getProvider("BC"); - // Tight bound on the default limit of 64MiB - final long fileSizeExceedingDefaultLimit = 1024 * 1024 * 64 + 1; - final InputStream largeObjectStream = new BoundedInputStream(fileSizeExceedingDefaultLimit); - v3Client.putObject(PutObjectRequest.builder() - .bucket(BUCKET) - .key(objectKey) - .build(), RequestBody.fromInputStream(largeObjectStream, fileSizeExceedingDefaultLimit)); + // V3 Client + S3Client v3Client = S3EncryptionClient.builder() + .aesKey(AES_KEY) + .cryptoProvider(provider) + .build(); - largeObjectStream.close(); + // Tight bound on the default limit of 64MiB + final long fileSizeExceedingDefaultLimit = 1024 * 1024 * 64 + 1; + final InputStream largeObjectStream = new BoundedInputStream(fileSizeExceedingDefaultLimit); + v3Client.putObject(PutObjectRequest.builder() + .bucket(BUCKET) + .key(objectKey) + .build(), RequestBody.fromInputStream(largeObjectStream, fileSizeExceedingDefaultLimit)); - // Delayed Authentication is not enabled, so getObject fails - assertThrows(S3EncryptionClientException.class, () -> v3Client.getObjectAsBytes(builder -> builder - .bucket(BUCKET) - .key(objectKey))); + largeObjectStream.close(); - S3Client v3ClientWithDelayedAuth = S3EncryptionClient.builder() - .aesKey(AES_KEY) - .enableDelayedAuthenticationMode(true) - .build(); + // Delayed Authentication is not enabled, so getObject fails + assertThrows(S3EncryptionClientException.class, () -> v3Client.getObjectAsBytes(builder -> builder + .bucket(BUCKET) + .key(objectKey))); - // Once enabled, the getObject request passes - ResponseInputStream response = v3ClientWithDelayedAuth.getObject(builder -> builder - .bucket(BUCKET) - .key(objectKey)); + S3Client v3ClientWithDelayedAuth = S3EncryptionClient.builder() + .aesKey(AES_KEY) + .enableDelayedAuthenticationMode(true) + .build(); + // Once enabled, the getObject request passes + ResponseInputStream response = v3ClientWithDelayedAuth.getObject(builder -> builder + .bucket(BUCKET) + .key(objectKey)); - assertTrue(IOUtils.contentEquals(new BoundedInputStream(fileSizeExceedingDefaultLimit), response)); - response.close(); - // Cleanup - deleteObject(BUCKET, objectKey, v3Client); - v3Client.close(); + assertTrue(IOUtils.contentEquals(new BoundedInputStream(fileSizeExceedingDefaultLimit), response)); + response.close(); - testCasePassed = true; + // Cleanup + deleteObject(BUCKET, objectKey, v3Client); + v3Client.close(); + } } @Test @@ -505,16 +477,16 @@ public void delayedAuthModeWithLargerThanMaxObjectFails() throws IOException { // V3 Client S3Client v3Client = S3EncryptionClient.builder() - .aesKey(AES_KEY) - .enableDelayedAuthenticationMode(true) - .build(); + .aesKey(AES_KEY) + .enableDelayedAuthenticationMode(true) + .build(); final long fileSizeExceedingGCMLimit = (1L << 39) - 256 / 8; final InputStream largeObjectStream = new BoundedInputStream(fileSizeExceedingGCMLimit); assertThrows(S3EncryptionClientException.class, () -> v3Client.putObject(PutObjectRequest.builder() - .bucket(BUCKET) - .key(objectKey) - .build(), RequestBody.fromInputStream(largeObjectStream, fileSizeExceedingGCMLimit))); + .bucket(BUCKET) + .key(objectKey) + .build(), RequestBody.fromInputStream(largeObjectStream, fileSizeExceedingGCMLimit))); largeObjectStream.close(); @@ -528,31 +500,31 @@ public void AesGcmV3toV3StreamWithTamperedTag() { // V3 Client S3Client v3Client = S3EncryptionClient.builder() - .aesKey(AES_KEY) - .build(); + .aesKey(AES_KEY) + .build(); // 640 bytes of gibberish - enough to cover multiple blocks final String input = "1esAAAYoAesAAAYoAesAAAYoAesAAAYoAesAAAYoAesAAAYoAesAAAYoAesAAAYo" - + "2esAAAYoAesAAAYoAesAAAYoAesAAAYoAesAAAYoAesAAAYoAesAAAYoAesAAAYo" - + "3esAAAYoAesAAAYoAesAAAYoAesAAAYoAesAAAYoAesAAAYoAesAAAYoAesAAAYo" - + "4esAAAYoAesAAAYoAesAAAYoAesAAAYoAesAAAYoAesAAAYoAesAAAYoAesAAAYo" - + "5esAAAYoAesAAAYoAesAAAYoAesAAAYoAesAAAYoAesAAAYoAesAAAYoAesAAAYo" - + "6esAAAYoAesAAAYoAesAAAYoAesAAAYoAesAAAYoAesAAAYoAesAAAYoAesAAAYo" - + "7esAAAYoAesAAAYoAesAAAYoAesAAAYoAesAAAYoAesAAAYoAesAAAYoAesAAAYo" - + "8esAAAYoAesAAAYoAesAAAYoAesAAAYoAesAAAYoAesAAAYoAesAAAYoAesAAAYo" - + "9esAAAYoAesAAAYoAesAAAYoAesAAAYoAesAAAYoAesAAAYoAesAAAYoAesAAAYo" - + "10sAAAYoAesAAAEndOfChunkAesAAAYoAesAAAYoAesAAAYoAesAAAYoAesAAAYo"; + + "2esAAAYoAesAAAYoAesAAAYoAesAAAYoAesAAAYoAesAAAYoAesAAAYoAesAAAYo" + + "3esAAAYoAesAAAYoAesAAAYoAesAAAYoAesAAAYoAesAAAYoAesAAAYoAesAAAYo" + + "4esAAAYoAesAAAYoAesAAAYoAesAAAYoAesAAAYoAesAAAYoAesAAAYoAesAAAYo" + + "5esAAAYoAesAAAYoAesAAAYoAesAAAYoAesAAAYoAesAAAYoAesAAAYoAesAAAYo" + + "6esAAAYoAesAAAYoAesAAAYoAesAAAYoAesAAAYoAesAAAYoAesAAAYoAesAAAYo" + + "7esAAAYoAesAAAYoAesAAAYoAesAAAYoAesAAAYoAesAAAYoAesAAAYoAesAAAYo" + + "8esAAAYoAesAAAYoAesAAAYoAesAAAYoAesAAAYoAesAAAYoAesAAAYoAesAAAYo" + + "9esAAAYoAesAAAYoAesAAAYoAesAAAYoAesAAAYoAesAAAYoAesAAAYoAesAAAYo" + + "10sAAAYoAesAAAEndOfChunkAesAAAYoAesAAAYoAesAAAYoAesAAAYoAesAAAYo"; final int inputLength = input.length(); v3Client.putObject(PutObjectRequest.builder() - .bucket(BUCKET) - .key(objectKey) - .build(), RequestBody.fromString(input)); + .bucket(BUCKET) + .key(objectKey) + .build(), RequestBody.fromString(input)); // Use an unencrypted (plaintext) client to interact with the encrypted object final S3Client plaintextS3Client = S3Client.builder().build(); ResponseBytes objectResponse = plaintextS3Client.getObjectAsBytes(builder -> builder - .bucket(BUCKET) - .key(objectKey)); + .bucket(BUCKET) + .key(objectKey)); final byte[] encryptedBytes = objectResponse.asByteArray(); final int tagLength = 16; final byte[] tamperedBytes = new byte[inputLength + tagLength]; @@ -571,16 +543,16 @@ public void AesGcmV3toV3StreamWithTamperedTag() { // Replace the encrypted object with the tampered object PutObjectRequest tamperedPut = PutObjectRequest.builder() - .bucket(BUCKET) - .key(objectKey) - .metadata(objectResponse.response().metadata()) // Preserve metadata from encrypted object - .build(); + .bucket(BUCKET) + .key(objectKey) + .metadata(objectResponse.response().metadata()) // Preserve metadata from encrypted object + .build(); plaintextS3Client.putObject(tamperedPut, RequestBody.fromBytes(tamperedBytes)); // Get (and decrypt) the (modified) object from S3 ResponseInputStream dataStream = v3Client.getObject(builder -> builder - .bucket(BUCKET) - .key(objectKey)); + .bucket(BUCKET) + .key(objectKey)); final int chunkSize = 300; final byte[] chunk1 = new byte[chunkSize]; @@ -599,4 +571,4 @@ public void AesGcmV3toV3StreamWithTamperedTag() { deleteObject(BUCKET, objectKey, v3Client); v3Client.close(); } -} +} \ No newline at end of file From 1271969a114fa0241b45a1649515c3f1e01cb8b8 Mon Sep 17 00:00:00 2001 From: Anirav Kareddy Date: Fri, 25 Jul 2025 11:03:44 -0700 Subject: [PATCH 11/26] trying mobile instead of cross-regison for smart configuration --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index a67f21ca2..82d2bec75 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -46,7 +46,7 @@ jobs: - name: Test run: | - export AWS_DEFAULTS_MODE=cross-region + export AWS_DEFAULTS_MODE=mobile export AWS_S3EC_TEST_ALT_KMS_KEY_ARN=arn:aws:kms:${{ vars.CI_AWS_REGION }}:${{ secrets.CI_AWS_ACCOUNT_ID }}:key/${{ vars.CI_ALT_KMS_KEY_ID }} export AWS_S3EC_TEST_ALT_ROLE_ARN=arn:aws:iam::${{ secrets.CI_AWS_ACCOUNT_ID }}:role/service-role/${{ vars.CI_ALT_ROLE }} export AWS_S3EC_TEST_BUCKET=${{ vars.CI_S3_BUCKET }} From 686e530bcf9c1a52e2fb2d268536f35f7cc17367 Mon Sep 17 00:00:00 2001 From: Anirav Kareddy Date: Fri, 25 Jul 2025 11:09:40 -0700 Subject: [PATCH 12/26] put the AWS_DEFAULTS_MODE inside env section for test --- .github/workflows/build.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 82d2bec75..df2d98ae6 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -45,8 +45,9 @@ jobs: shell: bash - name: Test + env: + AWS_DEFAULTS_MODE: mobile run: | - export AWS_DEFAULTS_MODE=mobile export AWS_S3EC_TEST_ALT_KMS_KEY_ARN=arn:aws:kms:${{ vars.CI_AWS_REGION }}:${{ secrets.CI_AWS_ACCOUNT_ID }}:key/${{ vars.CI_ALT_KMS_KEY_ID }} export AWS_S3EC_TEST_ALT_ROLE_ARN=arn:aws:iam::${{ secrets.CI_AWS_ACCOUNT_ID }}:role/service-role/${{ vars.CI_ALT_ROLE }} export AWS_S3EC_TEST_BUCKET=${{ vars.CI_S3_BUCKET }} From b83eec3e8471582d90fb2efa00138aa14a2fb934 Mon Sep 17 00:00:00 2001 From: Anirav Kareddy Date: Fri, 25 Jul 2025 11:50:36 -0700 Subject: [PATCH 13/26] checking to see if AWS_DEFAULTS_MODE is being set --- .github/workflows/build.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index df2d98ae6..7fbe6fa1b 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -48,6 +48,7 @@ jobs: env: AWS_DEFAULTS_MODE: mobile run: | + echo "AWS_DEFAULTS_MODE is set to: $AWS_DEFAULTS_MODE" export AWS_S3EC_TEST_ALT_KMS_KEY_ARN=arn:aws:kms:${{ vars.CI_AWS_REGION }}:${{ secrets.CI_AWS_ACCOUNT_ID }}:key/${{ vars.CI_ALT_KMS_KEY_ID }} export AWS_S3EC_TEST_ALT_ROLE_ARN=arn:aws:iam::${{ secrets.CI_AWS_ACCOUNT_ID }}:role/service-role/${{ vars.CI_ALT_ROLE }} export AWS_S3EC_TEST_BUCKET=${{ vars.CI_S3_BUCKET }} From 5d24f6e21459dabeab9d07e773d41e88297a6c3d Mon Sep 17 00:00:00 2001 From: Anirav Kareddy Date: Fri, 25 Jul 2025 11:54:07 -0700 Subject: [PATCH 14/26] added debug to see what is going on --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 7fbe6fa1b..53eb78daf 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -47,8 +47,8 @@ jobs: - name: Test env: AWS_DEFAULTS_MODE: mobile + AWS_SDK_CLIENT_LOGGING: debug run: | - echo "AWS_DEFAULTS_MODE is set to: $AWS_DEFAULTS_MODE" export AWS_S3EC_TEST_ALT_KMS_KEY_ARN=arn:aws:kms:${{ vars.CI_AWS_REGION }}:${{ secrets.CI_AWS_ACCOUNT_ID }}:key/${{ vars.CI_ALT_KMS_KEY_ID }} export AWS_S3EC_TEST_ALT_ROLE_ARN=arn:aws:iam::${{ secrets.CI_AWS_ACCOUNT_ID }}:role/service-role/${{ vars.CI_ALT_ROLE }} export AWS_S3EC_TEST_BUCKET=${{ vars.CI_S3_BUCKET }} From 8e07851f927b7bd58b982661de13a68798f87415 Mon Sep 17 00:00:00 2001 From: Anirav Kareddy Date: Tue, 5 Aug 2025 15:27:47 -0700 Subject: [PATCH 15/26] making stream support mark/reset --- .../encryption/s3/S3EncryptionClientStreamTest.java | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/src/test/java/software/amazon/encryption/s3/S3EncryptionClientStreamTest.java b/src/test/java/software/amazon/encryption/s3/S3EncryptionClientStreamTest.java index e228a22af..4f19cb30b 100644 --- a/src/test/java/software/amazon/encryption/s3/S3EncryptionClientStreamTest.java +++ b/src/test/java/software/amazon/encryption/s3/S3EncryptionClientStreamTest.java @@ -33,6 +33,7 @@ import javax.crypto.AEADBadTagException; import javax.crypto.KeyGenerator; import javax.crypto.SecretKey; +import java.io.BufferedInputStream; import java.io.IOException; import java.io.InputStream; import java.nio.charset.StandardCharsets; @@ -331,7 +332,9 @@ public void customSetBufferSizeWithLargeObject() throws IOException { // Tight bound on the custom buffer size limit of 32MiB final long fileSizeExceedingDefaultLimit = 1024 * 1024 * 32 + 1; - final InputStream largeObjectStream = new BoundedInputStream(fileSizeExceedingDefaultLimit); + final InputStream largeObjectStream = new BufferedInputStream( + new BoundedInputStream(fileSizeExceedingDefaultLimit) + ); v3ClientWithBuffer32MiB.putObject(PutObjectRequest.builder() .bucket(BUCKET) .key(objectKey) @@ -385,7 +388,9 @@ public void customSetBufferSizeWithLargeObjectAsyncClient() throws IOException { // Tight bound on the custom buffer size limit of 32MiB final long fileSizeExceedingDefaultLimit = 1024 * 1024 * 32 + 1; - final InputStream largeObjectStream = new BoundedInputStream(fileSizeExceedingDefaultLimit); + final InputStream largeObjectStream = new BufferedInputStream( + new BoundedInputStream(fileSizeExceedingDefaultLimit) + ); ExecutorService singleThreadExecutor = Executors.newSingleThreadExecutor(); CompletableFuture futurePut = v3ClientWithBuffer32MiB.putObject(PutObjectRequest.builder() .bucket(BUCKET) @@ -438,7 +443,9 @@ public void delayedAuthModeWithLargeObject() throws IOException { // Tight bound on the default limit of 64MiB final long fileSizeExceedingDefaultLimit = 1024 * 1024 * 64 + 1; - final InputStream largeObjectStream = new BoundedInputStream(fileSizeExceedingDefaultLimit); + final InputStream largeObjectStream = new BufferedInputStream( + new BoundedInputStream(fileSizeExceedingDefaultLimit) + ); v3Client.putObject(PutObjectRequest.builder() .bucket(BUCKET) .key(objectKey) From bf0a94ac0a489ccc04abf4fe729f19829f4eb85d Mon Sep 17 00:00:00 2001 From: Anirav Kareddy Date: Wed, 6 Aug 2025 12:05:48 -0700 Subject: [PATCH 16/26] trying byte array input stream approach --- .../s3/S3EncryptionClientStreamTest.java | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/src/test/java/software/amazon/encryption/s3/S3EncryptionClientStreamTest.java b/src/test/java/software/amazon/encryption/s3/S3EncryptionClientStreamTest.java index 4f19cb30b..8874fadc7 100644 --- a/src/test/java/software/amazon/encryption/s3/S3EncryptionClientStreamTest.java +++ b/src/test/java/software/amazon/encryption/s3/S3EncryptionClientStreamTest.java @@ -34,6 +34,7 @@ import javax.crypto.KeyGenerator; import javax.crypto.SecretKey; import java.io.BufferedInputStream; +import java.io.ByteArrayInputStream; import java.io.IOException; import java.io.InputStream; import java.nio.charset.StandardCharsets; @@ -332,9 +333,12 @@ public void customSetBufferSizeWithLargeObject() throws IOException { // Tight bound on the custom buffer size limit of 32MiB final long fileSizeExceedingDefaultLimit = 1024 * 1024 * 32 + 1; - final InputStream largeObjectStream = new BufferedInputStream( - new BoundedInputStream(fileSizeExceedingDefaultLimit) - ); + + byte[] data = new byte[(int) fileSizeExceedingDefaultLimit]; + for(int j=0; j < data.length; j++) { + data[i] = (byte) (j % 256); + } + final InputStream largeObjectStream = new ByteArrayInputStream(data); v3ClientWithBuffer32MiB.putObject(PutObjectRequest.builder() .bucket(BUCKET) .key(objectKey) @@ -352,8 +356,9 @@ public void customSetBufferSizeWithLargeObject() throws IOException { .bucket(BUCKET) .key(objectKey)); - - assertTrue(IOUtils.contentEquals(new BoundedInputStream(fileSizeExceedingDefaultLimit), response)); + // Create a new ByteArrayInputStream for comparison + InputStream expectedStream = new ByteArrayInputStream(data); + assertTrue(IOUtils.contentEquals(expectedStream, response)); response.close(); // Cleanup From d0ce7935ee46a38917b1b5c8d3c3411ea5c8e19b Mon Sep 17 00:00:00 2001 From: Anirav Kareddy Date: Wed, 6 Aug 2025 12:11:32 -0700 Subject: [PATCH 17/26] same approach of byte array input stream --- .../encryption/s3/S3EncryptionClientStreamTest.java | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/src/test/java/software/amazon/encryption/s3/S3EncryptionClientStreamTest.java b/src/test/java/software/amazon/encryption/s3/S3EncryptionClientStreamTest.java index 8874fadc7..8702f9d69 100644 --- a/src/test/java/software/amazon/encryption/s3/S3EncryptionClientStreamTest.java +++ b/src/test/java/software/amazon/encryption/s3/S3EncryptionClientStreamTest.java @@ -393,9 +393,11 @@ public void customSetBufferSizeWithLargeObjectAsyncClient() throws IOException { // Tight bound on the custom buffer size limit of 32MiB final long fileSizeExceedingDefaultLimit = 1024 * 1024 * 32 + 1; - final InputStream largeObjectStream = new BufferedInputStream( - new BoundedInputStream(fileSizeExceedingDefaultLimit) - ); + byte[] data = new byte[(int) fileSizeExceedingDefaultLimit]; + for(int j=0; j < data.length; j++) { + data[i] = (byte) (j % 256); + } + final InputStream largeObjectStream = new ByteArrayInputStream(data); ExecutorService singleThreadExecutor = Executors.newSingleThreadExecutor(); CompletableFuture futurePut = v3ClientWithBuffer32MiB.putObject(PutObjectRequest.builder() .bucket(BUCKET) @@ -422,7 +424,9 @@ public void customSetBufferSizeWithLargeObjectAsyncClient() throws IOException { .key(objectKey), AsyncResponseTransformer.toBlockingInputStream()); ResponseInputStream output = futureGet.join(); - assertTrue(IOUtils.contentEquals(new BoundedInputStream(fileSizeExceedingDefaultLimit), output)); + ByteArrayInputStream expectedStream = new ByteArrayInputStream(data); + assertTrue(IOUtils.contentEquals(expectedStream, output)); + output.close(); // Cleanup From a0d8ebb360e079a5ebce4cde1a879e3a9c487d62 Mon Sep 17 00:00:00 2001 From: Anirav Kareddy Date: Wed, 6 Aug 2025 12:15:56 -0700 Subject: [PATCH 18/26] just want to checkout 2 methods --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 53eb78daf..62e27e2ab 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -56,7 +56,7 @@ jobs: export AWS_S3EC_TEST_KMS_KEY_ID=arn:aws:kms:${{ vars.CI_AWS_REGION }}:${{ secrets.CI_AWS_ACCOUNT_ID }}:key/${{ vars.CI_KMS_KEY_ID }} export AWS_S3EC_TEST_KMS_KEY_ALIAS=arn:aws:kms:${{ vars.CI_AWS_REGION }}:${{ secrets.CI_AWS_ACCOUNT_ID }}:alias/${{ vars.CI_KMS_KEY_ALIAS }} export AWS_REGION=${{ vars.CI_AWS_REGION }} - mvn -B -ntp test -DskipCompile -Dtest=S3EncryptionClientStreamTest#customSetBufferSizeWithLargeObject,S3EncryptionClientStreamTest#customSetBufferSizeWithLargeObjectAsyncClient,S3EncryptionClientStreamTest#delayedAuthModeWithLargeObject + mvn -B -ntp test -DskipCompile -Dtest=S3EncryptionClientStreamTest#customSetBufferSizeWithLargeObject,S3EncryptionClientStreamTest#customSetBufferSizeWithLargeObjectAsyncClient shell: bash - name: Package JAR From 0cce9236c766d5c5cdb3f3724efbdeac7f2da7d3 Mon Sep 17 00:00:00 2001 From: Anirav Kareddy Date: Wed, 6 Aug 2025 12:35:36 -0700 Subject: [PATCH 19/26] updating AWS_DEFAULTS_MODE to be cross-region --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 62e27e2ab..174f3b505 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -46,7 +46,7 @@ jobs: - name: Test env: - AWS_DEFAULTS_MODE: mobile + AWS_DEFAULTS_MODE: cross-region AWS_SDK_CLIENT_LOGGING: debug run: | export AWS_S3EC_TEST_ALT_KMS_KEY_ARN=arn:aws:kms:${{ vars.CI_AWS_REGION }}:${{ secrets.CI_AWS_ACCOUNT_ID }}:key/${{ vars.CI_ALT_KMS_KEY_ID }} From 1f4769163ef3e4189e6bc137d84c4318bc46a4ce Mon Sep 17 00:00:00 2001 From: Anirav Kareddy Date: Wed, 6 Aug 2025 12:38:37 -0700 Subject: [PATCH 20/26] changing it back to mobile --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 174f3b505..62e27e2ab 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -46,7 +46,7 @@ jobs: - name: Test env: - AWS_DEFAULTS_MODE: cross-region + AWS_DEFAULTS_MODE: mobile AWS_SDK_CLIENT_LOGGING: debug run: | export AWS_S3EC_TEST_ALT_KMS_KEY_ARN=arn:aws:kms:${{ vars.CI_AWS_REGION }}:${{ secrets.CI_AWS_ACCOUNT_ID }}:key/${{ vars.CI_ALT_KMS_KEY_ID }} From 0a9f0ec160c6c7d394b9995a7226e6236c3fdbbb Mon Sep 17 00:00:00 2001 From: Anirav Kareddy Date: Wed, 6 Aug 2025 12:46:47 -0700 Subject: [PATCH 21/26] trying a new approach via retry_mode ad max_attempts --- .github/workflows/build.yml | 4 +- .../s3/S3EncryptionClientStreamTest.java | 194 +++++++++--------- 2 files changed, 98 insertions(+), 100 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 62e27e2ab..5353135e6 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -46,7 +46,9 @@ jobs: - name: Test env: - AWS_DEFAULTS_MODE: mobile + AWS_DEFAULTS_MODE: standard + AWS_RETRY_MODE: adaptive + AWS_MAX_ATTEMPTS: '10' AWS_SDK_CLIENT_LOGGING: debug run: | export AWS_S3EC_TEST_ALT_KMS_KEY_ARN=arn:aws:kms:${{ vars.CI_AWS_REGION }}:${{ secrets.CI_AWS_ACCOUNT_ID }}:key/${{ vars.CI_ALT_KMS_KEY_ID }} diff --git a/src/test/java/software/amazon/encryption/s3/S3EncryptionClientStreamTest.java b/src/test/java/software/amazon/encryption/s3/S3EncryptionClientStreamTest.java index 8702f9d69..0f7b38832 100644 --- a/src/test/java/software/amazon/encryption/s3/S3EncryptionClientStreamTest.java +++ b/src/test/java/software/amazon/encryption/s3/S3EncryptionClientStreamTest.java @@ -310,130 +310,126 @@ public void failsWhenBothBufferSizeAndDelayedAuthModeEnabled() { @Test public void customSetBufferSizeWithLargeObject() throws IOException { - for(int i=0; i < 10; i++) { - final String objectKey = appendTestSuffix("large-object-test-custom-buffer-size"); + final String objectKey = appendTestSuffix("large-object-test-custom-buffer-size"); - Security.addProvider(new BouncyCastleProvider()); - Provider provider = Security.getProvider("BC"); + Security.addProvider(new BouncyCastleProvider()); + Provider provider = Security.getProvider("BC"); - // V3 Client with custom max buffer size 32 MiB. - S3Client v3ClientWithBuffer32MiB = S3EncryptionClient.builder() - .aesKey(AES_KEY) - .cryptoProvider(provider) - .setBufferSize(32 * 1024 * 1024) - .build(); + // V3 Client with custom max buffer size 32 MiB. + S3Client v3ClientWithBuffer32MiB = S3EncryptionClient.builder() + .aesKey(AES_KEY) + .cryptoProvider(provider) + .setBufferSize(32 * 1024 * 1024) + .build(); - // V3 Client with default buffer size (i.e. 64MiB) - // When enableDelayedAuthenticationMode is set to true, delayed authentication mode always takes priority over buffered mode. - S3Client v3ClientWithDelayedAuth = S3EncryptionClient.builder() - .aesKey(AES_KEY) - .cryptoProvider(provider) - .enableDelayedAuthenticationMode(true) - .build(); + // V3 Client with default buffer size (i.e. 64MiB) + // When enableDelayedAuthenticationMode is set to true, delayed authentication mode always takes priority over buffered mode. + S3Client v3ClientWithDelayedAuth = S3EncryptionClient.builder() + .aesKey(AES_KEY) + .cryptoProvider(provider) + .enableDelayedAuthenticationMode(true) + .build(); - // Tight bound on the custom buffer size limit of 32MiB - final long fileSizeExceedingDefaultLimit = 1024 * 1024 * 32 + 1; + // Tight bound on the custom buffer size limit of 32MiB + final long fileSizeExceedingDefaultLimit = 1024 * 1024 * 32 + 1; - byte[] data = new byte[(int) fileSizeExceedingDefaultLimit]; - for(int j=0; j < data.length; j++) { - data[i] = (byte) (j % 256); - } - final InputStream largeObjectStream = new ByteArrayInputStream(data); - v3ClientWithBuffer32MiB.putObject(PutObjectRequest.builder() - .bucket(BUCKET) - .key(objectKey) - .build(), RequestBody.fromInputStream(largeObjectStream, fileSizeExceedingDefaultLimit)); + byte[] data = new byte[(int) fileSizeExceedingDefaultLimit]; + for(int j=0; j < data.length; j++) { + data[i] = (byte) (j % 256); + } + final InputStream largeObjectStream = new ByteArrayInputStream(data); + v3ClientWithBuffer32MiB.putObject(PutObjectRequest.builder() + .bucket(BUCKET) + .key(objectKey) + .build(), RequestBody.fromInputStream(largeObjectStream, fileSizeExceedingDefaultLimit)); - largeObjectStream.close(); + largeObjectStream.close(); - // Object is larger than Buffer, so getObject fails - assertThrows(S3EncryptionClientException.class, () -> v3ClientWithBuffer32MiB.getObjectAsBytes(builder -> builder - .bucket(BUCKET) - .key(objectKey))); + // Object is larger than Buffer, so getObject fails + assertThrows(S3EncryptionClientException.class, () -> v3ClientWithBuffer32MiB.getObjectAsBytes(builder -> builder + .bucket(BUCKET) + .key(objectKey))); - // You have to either enable the delayed auth mode or increase max buffer size (but in allowed bounds) - ResponseInputStream response = v3ClientWithDelayedAuth.getObject(builder -> builder - .bucket(BUCKET) - .key(objectKey)); + // You have to either enable the delayed auth mode or increase max buffer size (but in allowed bounds) + ResponseInputStream response = v3ClientWithDelayedAuth.getObject(builder -> builder + .bucket(BUCKET) + .key(objectKey)); - // Create a new ByteArrayInputStream for comparison - InputStream expectedStream = new ByteArrayInputStream(data); - assertTrue(IOUtils.contentEquals(expectedStream, response)); - response.close(); + // Create a new ByteArrayInputStream for comparison + InputStream expectedStream = new ByteArrayInputStream(data); + assertTrue(IOUtils.contentEquals(expectedStream, response)); + response.close(); - // Cleanup - deleteObject(BUCKET, objectKey, v3ClientWithBuffer32MiB); - v3ClientWithBuffer32MiB.close(); - v3ClientWithDelayedAuth.close(); - } + // Cleanup + deleteObject(BUCKET, objectKey, v3ClientWithBuffer32MiB); + v3ClientWithBuffer32MiB.close(); + v3ClientWithDelayedAuth.close(); } @Test public void customSetBufferSizeWithLargeObjectAsyncClient() throws IOException { - for(int i=0; i < 10;i++) { - final String objectKey = appendTestSuffix("large-object-test-custom-buffer-size-async"); + final String objectKey = appendTestSuffix("large-object-test-custom-buffer-size-async"); - Security.addProvider(new BouncyCastleProvider()); - Provider provider = Security.getProvider("BC"); + Security.addProvider(new BouncyCastleProvider()); + Provider provider = Security.getProvider("BC"); - // V3 Client with custom max buffer size 32 MiB. - S3AsyncClient v3ClientWithBuffer32MiB = S3AsyncEncryptionClient.builder() - .aesKey(AES_KEY) - .cryptoProvider(provider) - .setBufferSize(32 * 1024 * 1024) - .build(); - - // V3 Client with default buffer size (i.e. 64MiB) - // When enableDelayedAuthenticationMode is set to true, delayed authentication mode always takes priority over buffered mode. - S3AsyncClient v3ClientWithDelayedAuth = S3AsyncEncryptionClient.builder() - .aesKey(AES_KEY) - .cryptoProvider(provider) - .enableDelayedAuthenticationMode(true) - .build(); + // V3 Client with custom max buffer size 32 MiB. + S3AsyncClient v3ClientWithBuffer32MiB = S3AsyncEncryptionClient.builder() + .aesKey(AES_KEY) + .cryptoProvider(provider) + .setBufferSize(32 * 1024 * 1024) + .build(); - // Tight bound on the custom buffer size limit of 32MiB - final long fileSizeExceedingDefaultLimit = 1024 * 1024 * 32 + 1; - byte[] data = new byte[(int) fileSizeExceedingDefaultLimit]; - for(int j=0; j < data.length; j++) { - data[i] = (byte) (j % 256); - } - final InputStream largeObjectStream = new ByteArrayInputStream(data); - ExecutorService singleThreadExecutor = Executors.newSingleThreadExecutor(); - CompletableFuture futurePut = v3ClientWithBuffer32MiB.putObject(PutObjectRequest.builder() - .bucket(BUCKET) - .key(objectKey) - .build(), AsyncRequestBody.fromInputStream(largeObjectStream, fileSizeExceedingDefaultLimit, singleThreadExecutor)); + // V3 Client with default buffer size (i.e. 64MiB) + // When enableDelayedAuthenticationMode is set to true, delayed authentication mode always takes priority over buffered mode. + S3AsyncClient v3ClientWithDelayedAuth = S3AsyncEncryptionClient.builder() + .aesKey(AES_KEY) + .cryptoProvider(provider) + .enableDelayedAuthenticationMode(true) + .build(); - futurePut.join(); - largeObjectStream.close(); - singleThreadExecutor.shutdown(); + // Tight bound on the custom buffer size limit of 32MiB + final long fileSizeExceedingDefaultLimit = 1024 * 1024 * 32 + 1; + byte[] data = new byte[(int) fileSizeExceedingDefaultLimit]; + for(int j=0; j < data.length; j++) { + data[i] = (byte) (j % 256); + } + final InputStream largeObjectStream = new ByteArrayInputStream(data); + ExecutorService singleThreadExecutor = Executors.newSingleThreadExecutor(); + CompletableFuture futurePut = v3ClientWithBuffer32MiB.putObject(PutObjectRequest.builder() + .bucket(BUCKET) + .key(objectKey) + .build(), AsyncRequestBody.fromInputStream(largeObjectStream, fileSizeExceedingDefaultLimit, singleThreadExecutor)); - try { - // Object is larger than Buffer, so getObject fails - CompletableFuture> futureResponse = v3ClientWithBuffer32MiB.getObject(builder -> builder - .bucket(BUCKET) - .key(objectKey), AsyncResponseTransformer.toBlockingInputStream()); - futureResponse.join(); - } catch (CompletionException e) { - assertEquals(S3EncryptionClientException.class, e.getCause().getClass()); - } + futurePut.join(); + largeObjectStream.close(); + singleThreadExecutor.shutdown(); - // You have to either enable the delayed auth mode or increase max buffer size (but in allowed bounds) - CompletableFuture> futureGet = v3ClientWithDelayedAuth.getObject(builder -> builder + try { + // Object is larger than Buffer, so getObject fails + CompletableFuture> futureResponse = v3ClientWithBuffer32MiB.getObject(builder -> builder .bucket(BUCKET) .key(objectKey), AsyncResponseTransformer.toBlockingInputStream()); - ResponseInputStream output = futureGet.join(); + futureResponse.join(); + } catch (CompletionException e) { + assertEquals(S3EncryptionClientException.class, e.getCause().getClass()); + } + + // You have to either enable the delayed auth mode or increase max buffer size (but in allowed bounds) + CompletableFuture> futureGet = v3ClientWithDelayedAuth.getObject(builder -> builder + .bucket(BUCKET) + .key(objectKey), AsyncResponseTransformer.toBlockingInputStream()); + ResponseInputStream output = futureGet.join(); - ByteArrayInputStream expectedStream = new ByteArrayInputStream(data); - assertTrue(IOUtils.contentEquals(expectedStream, output)); + ByteArrayInputStream expectedStream = new ByteArrayInputStream(data); + assertTrue(IOUtils.contentEquals(expectedStream, output)); - output.close(); + output.close(); - // Cleanup - deleteObject(BUCKET, objectKey, v3ClientWithBuffer32MiB); - v3ClientWithBuffer32MiB.close(); - v3ClientWithDelayedAuth.close(); - } + // Cleanup + deleteObject(BUCKET, objectKey, v3ClientWithBuffer32MiB); + v3ClientWithBuffer32MiB.close(); + v3ClientWithDelayedAuth.close(); } @Test From 546e49b90f21536d4031df628aa1c8d8afcbe34d Mon Sep 17 00:00:00 2001 From: Anirav Kareddy Date: Wed, 6 Aug 2025 12:48:59 -0700 Subject: [PATCH 22/26] replaced i with j --- .../amazon/encryption/s3/S3EncryptionClientStreamTest.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/test/java/software/amazon/encryption/s3/S3EncryptionClientStreamTest.java b/src/test/java/software/amazon/encryption/s3/S3EncryptionClientStreamTest.java index 0f7b38832..8477fcb56 100644 --- a/src/test/java/software/amazon/encryption/s3/S3EncryptionClientStreamTest.java +++ b/src/test/java/software/amazon/encryption/s3/S3EncryptionClientStreamTest.java @@ -335,7 +335,7 @@ public void customSetBufferSizeWithLargeObject() throws IOException { byte[] data = new byte[(int) fileSizeExceedingDefaultLimit]; for(int j=0; j < data.length; j++) { - data[i] = (byte) (j % 256); + data[j] = (byte) (j % 256); } final InputStream largeObjectStream = new ByteArrayInputStream(data); v3ClientWithBuffer32MiB.putObject(PutObjectRequest.builder() @@ -392,7 +392,7 @@ public void customSetBufferSizeWithLargeObjectAsyncClient() throws IOException { final long fileSizeExceedingDefaultLimit = 1024 * 1024 * 32 + 1; byte[] data = new byte[(int) fileSizeExceedingDefaultLimit]; for(int j=0; j < data.length; j++) { - data[i] = (byte) (j % 256); + data[j] = (byte) (j % 256); } final InputStream largeObjectStream = new ByteArrayInputStream(data); ExecutorService singleThreadExecutor = Executors.newSingleThreadExecutor(); From b2dbba9cfab5f060b59aaceb294f9b6a8f6932d9 Mon Sep 17 00:00:00 2001 From: Anirav Kareddy Date: Wed, 6 Aug 2025 12:52:17 -0700 Subject: [PATCH 23/26] running each test case 10 times --- .../s3/S3EncryptionClientStreamTest.java | 194 +++++++++--------- 1 file changed, 99 insertions(+), 95 deletions(-) diff --git a/src/test/java/software/amazon/encryption/s3/S3EncryptionClientStreamTest.java b/src/test/java/software/amazon/encryption/s3/S3EncryptionClientStreamTest.java index 8477fcb56..e63a4e685 100644 --- a/src/test/java/software/amazon/encryption/s3/S3EncryptionClientStreamTest.java +++ b/src/test/java/software/amazon/encryption/s3/S3EncryptionClientStreamTest.java @@ -310,126 +310,130 @@ public void failsWhenBothBufferSizeAndDelayedAuthModeEnabled() { @Test public void customSetBufferSizeWithLargeObject() throws IOException { - final String objectKey = appendTestSuffix("large-object-test-custom-buffer-size"); + for(int i=0; i < 10; i++) { + final String objectKey = appendTestSuffix("large-object-test-custom-buffer-size"); - Security.addProvider(new BouncyCastleProvider()); - Provider provider = Security.getProvider("BC"); + Security.addProvider(new BouncyCastleProvider()); + Provider provider = Security.getProvider("BC"); - // V3 Client with custom max buffer size 32 MiB. - S3Client v3ClientWithBuffer32MiB = S3EncryptionClient.builder() - .aesKey(AES_KEY) - .cryptoProvider(provider) - .setBufferSize(32 * 1024 * 1024) - .build(); + // V3 Client with custom max buffer size 32 MiB. + S3Client v3ClientWithBuffer32MiB = S3EncryptionClient.builder() + .aesKey(AES_KEY) + .cryptoProvider(provider) + .setBufferSize(32 * 1024 * 1024) + .build(); - // V3 Client with default buffer size (i.e. 64MiB) - // When enableDelayedAuthenticationMode is set to true, delayed authentication mode always takes priority over buffered mode. - S3Client v3ClientWithDelayedAuth = S3EncryptionClient.builder() - .aesKey(AES_KEY) - .cryptoProvider(provider) - .enableDelayedAuthenticationMode(true) - .build(); + // V3 Client with default buffer size (i.e. 64MiB) + // When enableDelayedAuthenticationMode is set to true, delayed authentication mode always takes priority over buffered mode. + S3Client v3ClientWithDelayedAuth = S3EncryptionClient.builder() + .aesKey(AES_KEY) + .cryptoProvider(provider) + .enableDelayedAuthenticationMode(true) + .build(); - // Tight bound on the custom buffer size limit of 32MiB - final long fileSizeExceedingDefaultLimit = 1024 * 1024 * 32 + 1; + // Tight bound on the custom buffer size limit of 32MiB + final long fileSizeExceedingDefaultLimit = 1024 * 1024 * 32 + 1; - byte[] data = new byte[(int) fileSizeExceedingDefaultLimit]; - for(int j=0; j < data.length; j++) { - data[j] = (byte) (j % 256); - } - final InputStream largeObjectStream = new ByteArrayInputStream(data); - v3ClientWithBuffer32MiB.putObject(PutObjectRequest.builder() - .bucket(BUCKET) - .key(objectKey) - .build(), RequestBody.fromInputStream(largeObjectStream, fileSizeExceedingDefaultLimit)); + byte[] data = new byte[(int) fileSizeExceedingDefaultLimit]; + for(int j=0; j < data.length; j++) { + data[j] = (byte) (j % 256); + } + final InputStream largeObjectStream = new ByteArrayInputStream(data); + v3ClientWithBuffer32MiB.putObject(PutObjectRequest.builder() + .bucket(BUCKET) + .key(objectKey) + .build(), RequestBody.fromInputStream(largeObjectStream, fileSizeExceedingDefaultLimit)); - largeObjectStream.close(); + largeObjectStream.close(); - // Object is larger than Buffer, so getObject fails - assertThrows(S3EncryptionClientException.class, () -> v3ClientWithBuffer32MiB.getObjectAsBytes(builder -> builder - .bucket(BUCKET) - .key(objectKey))); + // Object is larger than Buffer, so getObject fails + assertThrows(S3EncryptionClientException.class, () -> v3ClientWithBuffer32MiB.getObjectAsBytes(builder -> builder + .bucket(BUCKET) + .key(objectKey))); - // You have to either enable the delayed auth mode or increase max buffer size (but in allowed bounds) - ResponseInputStream response = v3ClientWithDelayedAuth.getObject(builder -> builder - .bucket(BUCKET) - .key(objectKey)); + // You have to either enable the delayed auth mode or increase max buffer size (but in allowed bounds) + ResponseInputStream response = v3ClientWithDelayedAuth.getObject(builder -> builder + .bucket(BUCKET) + .key(objectKey)); - // Create a new ByteArrayInputStream for comparison - InputStream expectedStream = new ByteArrayInputStream(data); - assertTrue(IOUtils.contentEquals(expectedStream, response)); - response.close(); + // Create a new ByteArrayInputStream for comparison + InputStream expectedStream = new ByteArrayInputStream(data); + assertTrue(IOUtils.contentEquals(expectedStream, response)); + response.close(); - // Cleanup - deleteObject(BUCKET, objectKey, v3ClientWithBuffer32MiB); - v3ClientWithBuffer32MiB.close(); - v3ClientWithDelayedAuth.close(); + // Cleanup + deleteObject(BUCKET, objectKey, v3ClientWithBuffer32MiB); + v3ClientWithBuffer32MiB.close(); + v3ClientWithDelayedAuth.close(); + } } @Test public void customSetBufferSizeWithLargeObjectAsyncClient() throws IOException { - final String objectKey = appendTestSuffix("large-object-test-custom-buffer-size-async"); + for(int i=0; i < 10; i++) { + final String objectKey = appendTestSuffix("large-object-test-custom-buffer-size-async"); - Security.addProvider(new BouncyCastleProvider()); - Provider provider = Security.getProvider("BC"); + Security.addProvider(new BouncyCastleProvider()); + Provider provider = Security.getProvider("BC"); - // V3 Client with custom max buffer size 32 MiB. - S3AsyncClient v3ClientWithBuffer32MiB = S3AsyncEncryptionClient.builder() - .aesKey(AES_KEY) - .cryptoProvider(provider) - .setBufferSize(32 * 1024 * 1024) - .build(); + // V3 Client with custom max buffer size 32 MiB. + S3AsyncClient v3ClientWithBuffer32MiB = S3AsyncEncryptionClient.builder() + .aesKey(AES_KEY) + .cryptoProvider(provider) + .setBufferSize(32 * 1024 * 1024) + .build(); - // V3 Client with default buffer size (i.e. 64MiB) - // When enableDelayedAuthenticationMode is set to true, delayed authentication mode always takes priority over buffered mode. - S3AsyncClient v3ClientWithDelayedAuth = S3AsyncEncryptionClient.builder() - .aesKey(AES_KEY) - .cryptoProvider(provider) - .enableDelayedAuthenticationMode(true) - .build(); + // V3 Client with default buffer size (i.e. 64MiB) + // When enableDelayedAuthenticationMode is set to true, delayed authentication mode always takes priority over buffered mode. + S3AsyncClient v3ClientWithDelayedAuth = S3AsyncEncryptionClient.builder() + .aesKey(AES_KEY) + .cryptoProvider(provider) + .enableDelayedAuthenticationMode(true) + .build(); - // Tight bound on the custom buffer size limit of 32MiB - final long fileSizeExceedingDefaultLimit = 1024 * 1024 * 32 + 1; - byte[] data = new byte[(int) fileSizeExceedingDefaultLimit]; - for(int j=0; j < data.length; j++) { - data[j] = (byte) (j % 256); - } - final InputStream largeObjectStream = new ByteArrayInputStream(data); - ExecutorService singleThreadExecutor = Executors.newSingleThreadExecutor(); - CompletableFuture futurePut = v3ClientWithBuffer32MiB.putObject(PutObjectRequest.builder() - .bucket(BUCKET) - .key(objectKey) - .build(), AsyncRequestBody.fromInputStream(largeObjectStream, fileSizeExceedingDefaultLimit, singleThreadExecutor)); + // Tight bound on the custom buffer size limit of 32MiB + final long fileSizeExceedingDefaultLimit = 1024 * 1024 * 32 + 1; + byte[] data = new byte[(int) fileSizeExceedingDefaultLimit]; + for(int j=0; j < data.length; j++) { + data[j] = (byte) (j % 256); + } + final InputStream largeObjectStream = new ByteArrayInputStream(data); + ExecutorService singleThreadExecutor = Executors.newSingleThreadExecutor(); + CompletableFuture futurePut = v3ClientWithBuffer32MiB.putObject(PutObjectRequest.builder() + .bucket(BUCKET) + .key(objectKey) + .build(), AsyncRequestBody.fromInputStream(largeObjectStream, fileSizeExceedingDefaultLimit, singleThreadExecutor)); - futurePut.join(); - largeObjectStream.close(); - singleThreadExecutor.shutdown(); + futurePut.join(); + largeObjectStream.close(); + singleThreadExecutor.shutdown(); - try { - // Object is larger than Buffer, so getObject fails - CompletableFuture> futureResponse = v3ClientWithBuffer32MiB.getObject(builder -> builder + try { + // Object is larger than Buffer, so getObject fails + CompletableFuture> futureResponse = v3ClientWithBuffer32MiB.getObject(builder -> builder + .bucket(BUCKET) + .key(objectKey), AsyncResponseTransformer.toBlockingInputStream()); + futureResponse.join(); + } catch (CompletionException e) { + assertEquals(S3EncryptionClientException.class, e.getCause().getClass()); + } + + // You have to either enable the delayed auth mode or increase max buffer size (but in allowed bounds) + CompletableFuture> futureGet = v3ClientWithDelayedAuth.getObject(builder -> builder .bucket(BUCKET) .key(objectKey), AsyncResponseTransformer.toBlockingInputStream()); - futureResponse.join(); - } catch (CompletionException e) { - assertEquals(S3EncryptionClientException.class, e.getCause().getClass()); - } + ResponseInputStream output = futureGet.join(); - // You have to either enable the delayed auth mode or increase max buffer size (but in allowed bounds) - CompletableFuture> futureGet = v3ClientWithDelayedAuth.getObject(builder -> builder - .bucket(BUCKET) - .key(objectKey), AsyncResponseTransformer.toBlockingInputStream()); - ResponseInputStream output = futureGet.join(); + ByteArrayInputStream expectedStream = new ByteArrayInputStream(data); + assertTrue(IOUtils.contentEquals(expectedStream, output)); - ByteArrayInputStream expectedStream = new ByteArrayInputStream(data); - assertTrue(IOUtils.contentEquals(expectedStream, output)); + output.close(); - output.close(); - - // Cleanup - deleteObject(BUCKET, objectKey, v3ClientWithBuffer32MiB); - v3ClientWithBuffer32MiB.close(); - v3ClientWithDelayedAuth.close(); + // Cleanup + deleteObject(BUCKET, objectKey, v3ClientWithBuffer32MiB); + v3ClientWithBuffer32MiB.close(); + v3ClientWithDelayedAuth.close(); + } } @Test From 764ac4381ffc03a3c58b51cc1cdd7b693b86df70 Mon Sep 17 00:00:00 2001 From: Anirav Kareddy Date: Wed, 6 Aug 2025 15:14:20 -0700 Subject: [PATCH 24/26] running modified 100 times --- .../amazon/encryption/s3/S3EncryptionClientStreamTest.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/test/java/software/amazon/encryption/s3/S3EncryptionClientStreamTest.java b/src/test/java/software/amazon/encryption/s3/S3EncryptionClientStreamTest.java index e63a4e685..504ff04f9 100644 --- a/src/test/java/software/amazon/encryption/s3/S3EncryptionClientStreamTest.java +++ b/src/test/java/software/amazon/encryption/s3/S3EncryptionClientStreamTest.java @@ -310,7 +310,7 @@ public void failsWhenBothBufferSizeAndDelayedAuthModeEnabled() { @Test public void customSetBufferSizeWithLargeObject() throws IOException { - for(int i=0; i < 10; i++) { + for(int i=0; i < 100; i++) { final String objectKey = appendTestSuffix("large-object-test-custom-buffer-size"); Security.addProvider(new BouncyCastleProvider()); @@ -370,7 +370,7 @@ public void customSetBufferSizeWithLargeObject() throws IOException { @Test public void customSetBufferSizeWithLargeObjectAsyncClient() throws IOException { - for(int i=0; i < 10; i++) { + for(int i=0; i < 100; i++) { final String objectKey = appendTestSuffix("large-object-test-custom-buffer-size-async"); Security.addProvider(new BouncyCastleProvider()); From 37ad067b30cc0008e0dcd7dd4b4a21800ed2dc3e Mon Sep 17 00:00:00 2001 From: Anirav Kareddy Date: Wed, 6 Aug 2025 15:29:12 -0700 Subject: [PATCH 25/26] fixes --- .github/workflows/build.yml | 4 +- .../s3/S3EncryptionClientStreamTest.java | 217 +++++++++--------- 2 files changed, 115 insertions(+), 106 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 5353135e6..da47a3e6c 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -48,7 +48,7 @@ jobs: env: AWS_DEFAULTS_MODE: standard AWS_RETRY_MODE: adaptive - AWS_MAX_ATTEMPTS: '10' + AWS_MAX_ATTEMPTS: '20' AWS_SDK_CLIENT_LOGGING: debug run: | export AWS_S3EC_TEST_ALT_KMS_KEY_ARN=arn:aws:kms:${{ vars.CI_AWS_REGION }}:${{ secrets.CI_AWS_ACCOUNT_ID }}:key/${{ vars.CI_ALT_KMS_KEY_ID }} @@ -58,7 +58,7 @@ jobs: export AWS_S3EC_TEST_KMS_KEY_ID=arn:aws:kms:${{ vars.CI_AWS_REGION }}:${{ secrets.CI_AWS_ACCOUNT_ID }}:key/${{ vars.CI_KMS_KEY_ID }} export AWS_S3EC_TEST_KMS_KEY_ALIAS=arn:aws:kms:${{ vars.CI_AWS_REGION }}:${{ secrets.CI_AWS_ACCOUNT_ID }}:alias/${{ vars.CI_KMS_KEY_ALIAS }} export AWS_REGION=${{ vars.CI_AWS_REGION }} - mvn -B -ntp test -DskipCompile -Dtest=S3EncryptionClientStreamTest#customSetBufferSizeWithLargeObject,S3EncryptionClientStreamTest#customSetBufferSizeWithLargeObjectAsyncClient + mvn -B -ntp test -DskipCompile -Dtest=S3EncryptionClientStreamTest#customSetBufferSizeWithLargeObject,S3EncryptionClientStreamTest#customSetBufferSizeWithLargeObjectAsyncClient shell: bash - name: Package JAR diff --git a/src/test/java/software/amazon/encryption/s3/S3EncryptionClientStreamTest.java b/src/test/java/software/amazon/encryption/s3/S3EncryptionClientStreamTest.java index 504ff04f9..b061a9d93 100644 --- a/src/test/java/software/amazon/encryption/s3/S3EncryptionClientStreamTest.java +++ b/src/test/java/software/amazon/encryption/s3/S3EncryptionClientStreamTest.java @@ -334,11 +334,7 @@ public void customSetBufferSizeWithLargeObject() throws IOException { // Tight bound on the custom buffer size limit of 32MiB final long fileSizeExceedingDefaultLimit = 1024 * 1024 * 32 + 1; - byte[] data = new byte[(int) fileSizeExceedingDefaultLimit]; - for(int j=0; j < data.length; j++) { - data[j] = (byte) (j % 256); - } - final InputStream largeObjectStream = new ByteArrayInputStream(data); + final InputStream largeObjectStream = new MarkResetBoundedZerosInputStream(fileSizeExceedingDefaultLimit); v3ClientWithBuffer32MiB.putObject(PutObjectRequest.builder() .bucket(BUCKET) .key(objectKey) @@ -357,7 +353,7 @@ public void customSetBufferSizeWithLargeObject() throws IOException { .key(objectKey)); // Create a new ByteArrayInputStream for comparison - InputStream expectedStream = new ByteArrayInputStream(data); + InputStream expectedStream = new MarkResetBoundedZerosInputStream(fileSizeExceedingDefaultLimit); assertTrue(IOUtils.contentEquals(expectedStream, response)); response.close(); @@ -370,121 +366,134 @@ public void customSetBufferSizeWithLargeObject() throws IOException { @Test public void customSetBufferSizeWithLargeObjectAsyncClient() throws IOException { + int success=0, failures = 0; for(int i=0; i < 100; i++) { - final String objectKey = appendTestSuffix("large-object-test-custom-buffer-size-async"); - - Security.addProvider(new BouncyCastleProvider()); - Provider provider = Security.getProvider("BC"); - - // V3 Client with custom max buffer size 32 MiB. - S3AsyncClient v3ClientWithBuffer32MiB = S3AsyncEncryptionClient.builder() - .aesKey(AES_KEY) - .cryptoProvider(provider) - .setBufferSize(32 * 1024 * 1024) - .build(); - - // V3 Client with default buffer size (i.e. 64MiB) - // When enableDelayedAuthenticationMode is set to true, delayed authentication mode always takes priority over buffered mode. - S3AsyncClient v3ClientWithDelayedAuth = S3AsyncEncryptionClient.builder() - .aesKey(AES_KEY) - .cryptoProvider(provider) - .enableDelayedAuthenticationMode(true) - .build(); + try { + final String objectKey = appendTestSuffix("large-object-test-custom-buffer-size-async"); + + Security.addProvider(new BouncyCastleProvider()); + Provider provider = Security.getProvider("BC"); + + // V3 Client with custom max buffer size 32 MiB. + S3AsyncClient v3ClientWithBuffer32MiB = S3AsyncEncryptionClient.builder() + .aesKey(AES_KEY) + .cryptoProvider(provider) + .setBufferSize(32 * 1024 * 1024) + .build(); + + // V3 Client with default buffer size (i.e. 64MiB) + // When enableDelayedAuthenticationMode is set to true, delayed authentication mode always takes priority over buffered mode. + S3AsyncClient v3ClientWithDelayedAuth = S3AsyncEncryptionClient.builder() + .aesKey(AES_KEY) + .cryptoProvider(provider) + .enableDelayedAuthenticationMode(true) + .build(); + + // Tight bound on the custom buffer size limit of 32MiB + final long fileSizeExceedingDefaultLimit = 1024 * 1024 * 32 + 1; + final InputStream largeObjectStream = new MarkResetBoundedZerosInputStream(fileSizeExceedingDefaultLimit); + + ExecutorService singleThreadExecutor = Executors.newSingleThreadExecutor(); + CompletableFuture futurePut = v3ClientWithBuffer32MiB.putObject(PutObjectRequest.builder() + .bucket(BUCKET) + .key(objectKey) + .build(), AsyncRequestBody.fromInputStream(largeObjectStream, fileSizeExceedingDefaultLimit, singleThreadExecutor)); - // Tight bound on the custom buffer size limit of 32MiB - final long fileSizeExceedingDefaultLimit = 1024 * 1024 * 32 + 1; - byte[] data = new byte[(int) fileSizeExceedingDefaultLimit]; - for(int j=0; j < data.length; j++) { - data[j] = (byte) (j % 256); - } - final InputStream largeObjectStream = new ByteArrayInputStream(data); - ExecutorService singleThreadExecutor = Executors.newSingleThreadExecutor(); - CompletableFuture futurePut = v3ClientWithBuffer32MiB.putObject(PutObjectRequest.builder() - .bucket(BUCKET) - .key(objectKey) - .build(), AsyncRequestBody.fromInputStream(largeObjectStream, fileSizeExceedingDefaultLimit, singleThreadExecutor)); + futurePut.join(); + largeObjectStream.close(); + singleThreadExecutor.shutdown(); - futurePut.join(); - largeObjectStream.close(); - singleThreadExecutor.shutdown(); + try { + // Object is larger than Buffer, so getObject fails + CompletableFuture> futureResponse = v3ClientWithBuffer32MiB.getObject(builder -> builder + .bucket(BUCKET) + .key(objectKey), AsyncResponseTransformer.toBlockingInputStream()); + futureResponse.join(); + } catch (CompletionException e) { + assertEquals(S3EncryptionClientException.class, e.getCause().getClass()); + } - try { - // Object is larger than Buffer, so getObject fails - CompletableFuture> futureResponse = v3ClientWithBuffer32MiB.getObject(builder -> builder + // You have to either enable the delayed auth mode or increase max buffer size (but in allowed bounds) + CompletableFuture> futureGet = v3ClientWithDelayedAuth.getObject(builder -> builder .bucket(BUCKET) .key(objectKey), AsyncResponseTransformer.toBlockingInputStream()); - futureResponse.join(); - } catch (CompletionException e) { - assertEquals(S3EncryptionClientException.class, e.getCause().getClass()); - } + ResponseInputStream output = futureGet.join(); - // You have to either enable the delayed auth mode or increase max buffer size (but in allowed bounds) - CompletableFuture> futureGet = v3ClientWithDelayedAuth.getObject(builder -> builder - .bucket(BUCKET) - .key(objectKey), AsyncResponseTransformer.toBlockingInputStream()); - ResponseInputStream output = futureGet.join(); + InputStream expectedStream = new MarkResetBoundedZerosInputStream(fileSizeExceedingDefaultLimit); + assertTrue(IOUtils.contentEquals(expectedStream, output)); - ByteArrayInputStream expectedStream = new ByteArrayInputStream(data); - assertTrue(IOUtils.contentEquals(expectedStream, output)); + output.close(); - output.close(); + // Cleanup + deleteObject(BUCKET, objectKey, v3ClientWithBuffer32MiB); + v3ClientWithBuffer32MiB.close(); + v3ClientWithDelayedAuth.close(); - // Cleanup - deleteObject(BUCKET, objectKey, v3ClientWithBuffer32MiB); - v3ClientWithBuffer32MiB.close(); - v3ClientWithDelayedAuth.close(); + success++; + } catch (Exception e) { + failures++; + } } + System.out.println("Success: "+success+" Failures: "+failures); } @Test public void delayedAuthModeWithLargeObject() throws IOException { + int success = 0, failures = 0; for(int i=0; i < 10; i++) { - final String objectKey = appendTestSuffix("large-object-test"); - - Security.addProvider(new BouncyCastleProvider()); - Provider provider = Security.getProvider("BC"); - - // V3 Client - S3Client v3Client = S3EncryptionClient.builder() - .aesKey(AES_KEY) - .cryptoProvider(provider) - .build(); - - // Tight bound on the default limit of 64MiB - final long fileSizeExceedingDefaultLimit = 1024 * 1024 * 64 + 1; - final InputStream largeObjectStream = new BufferedInputStream( - new BoundedInputStream(fileSizeExceedingDefaultLimit) - ); - v3Client.putObject(PutObjectRequest.builder() - .bucket(BUCKET) - .key(objectKey) - .build(), RequestBody.fromInputStream(largeObjectStream, fileSizeExceedingDefaultLimit)); - - largeObjectStream.close(); - - // Delayed Authentication is not enabled, so getObject fails - assertThrows(S3EncryptionClientException.class, () -> v3Client.getObjectAsBytes(builder -> builder - .bucket(BUCKET) - .key(objectKey))); - - S3Client v3ClientWithDelayedAuth = S3EncryptionClient.builder() - .aesKey(AES_KEY) - .enableDelayedAuthenticationMode(true) - .build(); - - // Once enabled, the getObject request passes - ResponseInputStream response = v3ClientWithDelayedAuth.getObject(builder -> builder - .bucket(BUCKET) - .key(objectKey)); - - - assertTrue(IOUtils.contentEquals(new BoundedInputStream(fileSizeExceedingDefaultLimit), response)); - response.close(); - - // Cleanup - deleteObject(BUCKET, objectKey, v3Client); - v3Client.close(); + try { + final String objectKey = appendTestSuffix("large-object-test"); + + Security.addProvider(new BouncyCastleProvider()); + Provider provider = Security.getProvider("BC"); + + // V3 Client + S3Client v3Client = S3EncryptionClient.builder() + .aesKey(AES_KEY) + .cryptoProvider(provider) + .build(); + + // Tight bound on the default limit of 64MiB + final long fileSizeExceedingDefaultLimit = 1024 * 1024 * 64 + 1; + final InputStream largeObjectStream = new BufferedInputStream( + new BoundedInputStream(fileSizeExceedingDefaultLimit) + ); + v3Client.putObject(PutObjectRequest.builder() + .bucket(BUCKET) + .key(objectKey) + .build(), RequestBody.fromInputStream(largeObjectStream, fileSizeExceedingDefaultLimit)); + + largeObjectStream.close(); + + // Delayed Authentication is not enabled, so getObject fails + assertThrows(S3EncryptionClientException.class, () -> v3Client.getObjectAsBytes(builder -> builder + .bucket(BUCKET) + .key(objectKey))); + + S3Client v3ClientWithDelayedAuth = S3EncryptionClient.builder() + .aesKey(AES_KEY) + .enableDelayedAuthenticationMode(true) + .build(); + + // Once enabled, the getObject request passes + ResponseInputStream response = v3ClientWithDelayedAuth.getObject(builder -> builder + .bucket(BUCKET) + .key(objectKey)); + + + assertTrue(IOUtils.contentEquals(new BoundedInputStream(fileSizeExceedingDefaultLimit), response)); + response.close(); + + // Cleanup + deleteObject(BUCKET, objectKey, v3Client); + v3Client.close(); + + success++; + } catch (Exception e) { + failures++; + } } + System.out.println("Success: "+success+" Failures: "+failures); } @Test From e3afcecf73720d2ab62dddc5bd2190e3d4ad2c6d Mon Sep 17 00:00:00 2001 From: Anirav Kareddy Date: Wed, 6 Aug 2025 16:51:42 -0700 Subject: [PATCH 26/26] testing (removed the array byte input stream to see if needed or not) --- .github/workflows/build.yml | 2 +- ...S3EncryptionClientInstructionFileTest.java | 373 +++++++++--------- ...S3EncryptionClientMultipartUploadTest.java | 327 +++++++-------- .../s3/S3EncryptionClientStreamTest.java | 111 +++--- .../examples/MultipartUploadExampleTest.java | 24 +- 5 files changed, 422 insertions(+), 415 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index da47a3e6c..070ff0f12 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -58,7 +58,7 @@ jobs: export AWS_S3EC_TEST_KMS_KEY_ID=arn:aws:kms:${{ vars.CI_AWS_REGION }}:${{ secrets.CI_AWS_ACCOUNT_ID }}:key/${{ vars.CI_KMS_KEY_ID }} export AWS_S3EC_TEST_KMS_KEY_ALIAS=arn:aws:kms:${{ vars.CI_AWS_REGION }}:${{ secrets.CI_AWS_ACCOUNT_ID }}:alias/${{ vars.CI_KMS_KEY_ALIAS }} export AWS_REGION=${{ vars.CI_AWS_REGION }} - mvn -B -ntp test -DskipCompile -Dtest=S3EncryptionClientStreamTest#customSetBufferSizeWithLargeObject,S3EncryptionClientStreamTest#customSetBufferSizeWithLargeObjectAsyncClient + mvn -B -ntp test -DskipCompile -Dtest=S3EncryptionClientStreamTest#customSetBufferSizeWithLargeObject,S3EncryptionClientStreamTest#customSetBufferSizeWithLargeObjectAsyncClient,S3EncryptionClientStreamTest#delayedAuthModeWithLargeObject,S3EncryptionClientInstructionFileTest#testMultipartPutWithInstructionFile,S3EncryptionClientMultipartUploadTest#multipartPutObject,S3EncryptionClientMultipartUploadTest#multipartPutObjectAsync,MultipartUploadExampleTest#testMultipartUploadExamples shell: bash - name: Package JAR diff --git a/src/test/java/software/amazon/encryption/s3/S3EncryptionClientInstructionFileTest.java b/src/test/java/software/amazon/encryption/s3/S3EncryptionClientInstructionFileTest.java index 680a6fc23..30977dc21 100644 --- a/src/test/java/software/amazon/encryption/s3/S3EncryptionClientInstructionFileTest.java +++ b/src/test/java/software/amazon/encryption/s3/S3EncryptionClientInstructionFileTest.java @@ -10,8 +10,6 @@ import com.amazonaws.services.s3.model.KMSEncryptionMaterials; import com.amazonaws.services.s3.model.StaticEncryptionMaterialsProvider; import org.apache.commons.io.IOUtils; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.RepeatedTest; import org.junit.jupiter.api.Test; import software.amazon.awssdk.core.ResponseBytes; import software.amazon.awssdk.core.ResponseInputStream; @@ -53,12 +51,6 @@ import static software.amazon.encryption.s3.utils.S3EncryptionClientTestResources.deleteObject; public class S3EncryptionClientInstructionFileTest { - private static boolean testCasePassed = false; - - @BeforeEach - public void resetTestCasePassedFlag() { - testCasePassed = false; - } @Test public void testInstructionFileExists() { @@ -66,32 +58,32 @@ public void testInstructionFileExists() { final String input = "SimpleTestOfV3EncryptionClient"; S3Client wrappedClient = S3Client.create(); S3Client s3Client = S3EncryptionClient.builder() - .instructionFileConfig(InstructionFileConfig.builder() - .instructionFileClient(wrappedClient) - .enableInstructionFilePutObject(true) - .build()) - .kmsKeyId(KMS_KEY_ID) - .build(); + .instructionFileConfig(InstructionFileConfig.builder() + .instructionFileClient(wrappedClient) + .enableInstructionFilePutObject(true) + .build()) + .kmsKeyId(KMS_KEY_ID) + .build(); s3Client.putObject(builder -> builder - .bucket(BUCKET) - .key(objectKey) - .build(), RequestBody.fromString(input)); + .bucket(BUCKET) + .key(objectKey) + .build(), RequestBody.fromString(input)); // Get the instruction file separately using a default client S3Client defaultClient = S3Client.create(); ResponseBytes directInstGetResponse = defaultClient.getObjectAsBytes(builder -> builder - .bucket(BUCKET) - .key(objectKey + ".instruction") - .build()); + .bucket(BUCKET) + .key(objectKey + ".instruction") + .build()); // Ensure its metadata identifies it as such assertTrue(directInstGetResponse.response().metadata().containsKey("x-amz-crypto-instr-file")); // Ensure decryption succeeds ResponseBytes objectResponse = s3Client.getObjectAsBytes(builder -> builder - .bucket(BUCKET) - .key(objectKey) - .build()); + .bucket(BUCKET) + .key(objectKey) + .build()); String output = objectResponse.asUtf8String(); assertEquals(input, output); @@ -106,33 +98,33 @@ public void testDisabledClientFails() { final String input = "SimpleTestOfV3EncryptionClient"; S3Client wrappedClient = S3Client.create(); S3Client s3Client = S3EncryptionClient.builder() - .instructionFileConfig(InstructionFileConfig.builder() - .instructionFileClient(wrappedClient) - .enableInstructionFilePutObject(true) - .build()) - .kmsKeyId(KMS_KEY_ID) - .build(); + .instructionFileConfig(InstructionFileConfig.builder() + .instructionFileClient(wrappedClient) + .enableInstructionFilePutObject(true) + .build()) + .kmsKeyId(KMS_KEY_ID) + .build(); // Put with Instruction File s3Client.putObject(builder -> builder - .bucket(BUCKET) - .key(objectKey) - .build(), RequestBody.fromString(input)); + .bucket(BUCKET) + .key(objectKey) + .build(), RequestBody.fromString(input)); // Disabled client should fail S3Client s3ClientDisabledInstructionFile = S3EncryptionClient.builder() - .wrappedClient(wrappedClient) - .instructionFileConfig(InstructionFileConfig.builder() - .disableInstructionFile(true) - .build()) - .kmsKeyId(KMS_KEY_ID) - .build(); + .wrappedClient(wrappedClient) + .instructionFileConfig(InstructionFileConfig.builder() + .disableInstructionFile(true) + .build()) + .kmsKeyId(KMS_KEY_ID) + .build(); try { s3ClientDisabledInstructionFile.getObjectAsBytes(builder -> builder - .bucket(BUCKET) - .key(objectKey) - .build()); + .bucket(BUCKET) + .key(objectKey) + .build()); fail("expected exception"); } catch (S3EncryptionClientException exception) { assertTrue(exception.getMessage().contains("Exception encountered while fetching Instruction File.")); @@ -154,32 +146,32 @@ public void testInstructionFileDelete() { final String input = "SimpleTestOfV3EncryptionClient"; S3Client wrappedClient = S3Client.create(); S3Client s3Client = S3EncryptionClient.builder() - .instructionFileConfig(InstructionFileConfig.builder() - .instructionFileClient(wrappedClient) - .enableInstructionFilePutObject(true) - .build()) - .kmsKeyId(KMS_KEY_ID) - .build(); + .instructionFileConfig(InstructionFileConfig.builder() + .instructionFileClient(wrappedClient) + .enableInstructionFilePutObject(true) + .build()) + .kmsKeyId(KMS_KEY_ID) + .build(); s3Client.putObject(builder -> builder - .bucket(BUCKET) - .key(objectKey) - .build(), RequestBody.fromString(input)); + .bucket(BUCKET) + .key(objectKey) + .build(), RequestBody.fromString(input)); // Get the instruction file separately using a default client S3Client defaultClient = S3Client.create(); ResponseBytes directInstGetResponse = defaultClient.getObjectAsBytes(builder -> builder - .bucket(BUCKET) - .key(objectKey + ".instruction") - .build()); + .bucket(BUCKET) + .key(objectKey + ".instruction") + .build()); // Ensure its metadata identifies it as such assertTrue(directInstGetResponse.response().metadata().containsKey("x-amz-crypto-instr-file")); // Ensure decryption succeeds ResponseBytes objectResponse = s3Client.getObjectAsBytes(builder -> builder - .bucket(BUCKET) - .key(objectKey) - .build()); + .bucket(BUCKET) + .key(objectKey) + .build()); String output = objectResponse.asUtf8String(); assertEquals(input, output); @@ -187,9 +179,9 @@ public void testInstructionFileDelete() { try { defaultClient.getObjectAsBytes(builder -> builder - .bucket(BUCKET) - .key(objectKey + ".instruction") - .build()); + .bucket(BUCKET) + .key(objectKey + ".instruction") + .build()); fail("expected exception!"); } catch (NoSuchKeyException e) { // expected @@ -205,28 +197,28 @@ public void testPutWithInstructionFileV3ToV2Kms() { final String input = "SimpleTestOfV3EncryptionClient"; S3Client wrappedClient = S3Client.create(); S3Client s3Client = S3EncryptionClient.builder() - .instructionFileConfig(InstructionFileConfig.builder() - .instructionFileClient(wrappedClient) - .enableInstructionFilePutObject(true) - .build()) - .kmsKeyId(KMS_KEY_ID) - .build(); + .instructionFileConfig(InstructionFileConfig.builder() + .instructionFileClient(wrappedClient) + .enableInstructionFilePutObject(true) + .build()) + .kmsKeyId(KMS_KEY_ID) + .build(); s3Client.putObject(builder -> builder - .bucket(BUCKET) - .key(objectKey) - .build(), RequestBody.fromString(input)); + .bucket(BUCKET) + .key(objectKey) + .build(), RequestBody.fromString(input)); EncryptionMaterialsProvider materialsProvider = - new StaticEncryptionMaterialsProvider(new KMSEncryptionMaterials(KMS_KEY_ID)); + new StaticEncryptionMaterialsProvider(new KMSEncryptionMaterials(KMS_KEY_ID)); CryptoConfigurationV2 cryptoConfig = - new CryptoConfigurationV2(CryptoMode.StrictAuthenticatedEncryption) - .withStorageMode(CryptoStorageMode.InstructionFile); + new CryptoConfigurationV2(CryptoMode.StrictAuthenticatedEncryption) + .withStorageMode(CryptoStorageMode.InstructionFile); AmazonS3EncryptionV2 v2Client = AmazonS3EncryptionClientV2.encryptionBuilder() - .withCryptoConfiguration(cryptoConfig) - .withEncryptionMaterialsProvider(materialsProvider) - .build(); + .withCryptoConfiguration(cryptoConfig) + .withEncryptionMaterialsProvider(materialsProvider) + .build(); String result = v2Client.getObjectAsString(BUCKET, objectKey); assertEquals(input, result); @@ -245,28 +237,28 @@ public void testPutWithInstructionFileV3ToV2Aes() throws NoSuchAlgorithmExceptio final String input = "SimpleTestOfV3EncryptionClient"; S3Client wrappedClient = S3Client.create(); S3Client s3Client = S3EncryptionClient.builder() - .instructionFileConfig(InstructionFileConfig.builder() - .instructionFileClient(wrappedClient) - .enableInstructionFilePutObject(true) - .build()) - .aesKey(aesKey) - .build(); + .instructionFileConfig(InstructionFileConfig.builder() + .instructionFileClient(wrappedClient) + .enableInstructionFilePutObject(true) + .build()) + .aesKey(aesKey) + .build(); s3Client.putObject(builder -> builder - .bucket(BUCKET) - .key(objectKey) - .build(), RequestBody.fromString(input)); + .bucket(BUCKET) + .key(objectKey) + .build(), RequestBody.fromString(input)); EncryptionMaterialsProvider materialsProvider = - new StaticEncryptionMaterialsProvider(new EncryptionMaterials(aesKey)); + new StaticEncryptionMaterialsProvider(new EncryptionMaterials(aesKey)); CryptoConfigurationV2 cryptoConfig = - new CryptoConfigurationV2(CryptoMode.StrictAuthenticatedEncryption) - .withStorageMode(CryptoStorageMode.InstructionFile); + new CryptoConfigurationV2(CryptoMode.StrictAuthenticatedEncryption) + .withStorageMode(CryptoStorageMode.InstructionFile); AmazonS3EncryptionV2 v2Client = AmazonS3EncryptionClientV2.encryptionBuilder() - .withCryptoConfiguration(cryptoConfig) - .withEncryptionMaterialsProvider(materialsProvider) - .build(); + .withCryptoConfiguration(cryptoConfig) + .withEncryptionMaterialsProvider(materialsProvider) + .build(); String result = v2Client.getObjectAsString(BUCKET, objectKey); assertEquals(input, result); @@ -286,28 +278,28 @@ public void testPutWithInstructionFileV3ToV2Rsa() throws NoSuchAlgorithmExceptio final String input = "SimpleTestOfV3EncryptionClient"; S3Client wrappedClient = S3Client.create(); S3Client s3Client = S3EncryptionClient.builder() - .instructionFileConfig(InstructionFileConfig.builder() - .instructionFileClient(wrappedClient) - .enableInstructionFilePutObject(true) - .build()) - .rsaKeyPair(rsaKey) - .build(); + .instructionFileConfig(InstructionFileConfig.builder() + .instructionFileClient(wrappedClient) + .enableInstructionFilePutObject(true) + .build()) + .rsaKeyPair(rsaKey) + .build(); s3Client.putObject(builder -> builder - .bucket(BUCKET) - .key(objectKey) - .build(), RequestBody.fromString(input)); + .bucket(BUCKET) + .key(objectKey) + .build(), RequestBody.fromString(input)); EncryptionMaterialsProvider materialsProvider = - new StaticEncryptionMaterialsProvider(new EncryptionMaterials(rsaKey)); + new StaticEncryptionMaterialsProvider(new EncryptionMaterials(rsaKey)); CryptoConfigurationV2 cryptoConfig = - new CryptoConfigurationV2(CryptoMode.StrictAuthenticatedEncryption) - .withStorageMode(CryptoStorageMode.InstructionFile); + new CryptoConfigurationV2(CryptoMode.StrictAuthenticatedEncryption) + .withStorageMode(CryptoStorageMode.InstructionFile); AmazonS3EncryptionV2 v2Client = AmazonS3EncryptionClientV2.encryptionBuilder() - .withCryptoConfiguration(cryptoConfig) - .withEncryptionMaterialsProvider(materialsProvider) - .build(); + .withCryptoConfiguration(cryptoConfig) + .withEncryptionMaterialsProvider(materialsProvider) + .build(); String result = v2Client.getObjectAsString(BUCKET, objectKey); assertEquals(input, result); @@ -317,52 +309,62 @@ public void testPutWithInstructionFileV3ToV2Rsa() throws NoSuchAlgorithmExceptio s3Client.close(); } - @RepeatedTest(3) + @Test public void testMultipartPutWithInstructionFile() throws IOException { - final String object_key = appendTestSuffix("test-multipart-put-instruction-file"); - - final long fileSizeLimit = 1024 * 1024 * 50; //50 MB - final InputStream inputStream = new BoundedInputStream(fileSizeLimit); - final InputStream objectStreamForResult = new BoundedInputStream(fileSizeLimit); - final StorageClass storageClass = StorageClass.STANDARD_IA; - - S3Client wrappedClient = S3Client.create(); - S3Client s3Client = S3EncryptionClient.builder() - .instructionFileConfig(InstructionFileConfig.builder() - .instructionFileClient(wrappedClient) - .enableInstructionFilePutObject(true) - .build()) - .kmsKeyId(KMS_KEY_ID) - .enableMultipartPutObject(true) - .build(); - - Map encryptionContext = new HashMap<>(); - encryptionContext.put("test-key", "test-value"); - - - s3Client.putObject(builder -> builder - .bucket(BUCKET) - .storageClass(storageClass) - .overrideConfiguration(withAdditionalConfiguration(encryptionContext)) - .key(object_key), RequestBody.fromInputStream(inputStream, fileSizeLimit)); - - S3Client defaultClient = S3Client.create(); - ResponseBytes directInstGetResponse = defaultClient.getObjectAsBytes(builder -> builder - .bucket(BUCKET) - .key(object_key + ".instruction") - .build()); - assertTrue(directInstGetResponse.response().metadata().containsKey("x-amz-crypto-instr-file")); - assertEquals(storageClass.toString(), directInstGetResponse.response().storageClassAsString()); - - ResponseInputStream getResponse = s3Client.getObject(builder -> builder - .bucket(BUCKET) - .overrideConfiguration(withAdditionalConfiguration(encryptionContext)) - .key(object_key)); - - assertTrue(IOUtils.contentEquals(objectStreamForResult, getResponse)); - - deleteObject(BUCKET, object_key, s3Client); - s3Client.close(); + int success = 0, failures = 0; + for(int i=0; i < 100; i++) { + try { + final String object_key = appendTestSuffix("test-multipart-put-instruction-file"); + + final long fileSizeLimit = 1024 * 1024 * 50; //50 MB + final InputStream inputStream = new BoundedInputStream(fileSizeLimit); + final InputStream objectStreamForResult = new BoundedInputStream(fileSizeLimit); + final StorageClass storageClass = StorageClass.STANDARD_IA; + + S3Client wrappedClient = S3Client.create(); + S3Client s3Client = S3EncryptionClient.builder() + .instructionFileConfig(InstructionFileConfig.builder() + .instructionFileClient(wrappedClient) + .enableInstructionFilePutObject(true) + .build()) + .kmsKeyId(KMS_KEY_ID) + .enableMultipartPutObject(true) + .build(); + + Map encryptionContext = new HashMap<>(); + encryptionContext.put("test-key", "test-value"); + + + s3Client.putObject(builder -> builder + .bucket(BUCKET) + .storageClass(storageClass) + .overrideConfiguration(withAdditionalConfiguration(encryptionContext)) + .key(object_key), RequestBody.fromInputStream(inputStream, fileSizeLimit)); + + S3Client defaultClient = S3Client.create(); + ResponseBytes directInstGetResponse = defaultClient.getObjectAsBytes(builder -> builder + .bucket(BUCKET) + .key(object_key + ".instruction") + .build()); + assertTrue(directInstGetResponse.response().metadata().containsKey("x-amz-crypto-instr-file")); + assertEquals(storageClass.toString(), directInstGetResponse.response().storageClassAsString()); + + ResponseInputStream getResponse = s3Client.getObject(builder -> builder + .bucket(BUCKET) + .overrideConfiguration(withAdditionalConfiguration(encryptionContext)) + .key(object_key)); + + assertTrue(IOUtils.contentEquals(objectStreamForResult, getResponse)); + + deleteObject(BUCKET, object_key, s3Client); + s3Client.close(); + + success++; + } catch (Exception e) { + failures++; + } + } + System.out.println("testMultipartPutWithInstructionFile: Success: "+success+" Failures: "+failures); } @@ -383,17 +385,17 @@ public void testLowLevelMultipartPutWithInstructionFile() throws NoSuchAlgorithm S3Client wrappedClient = S3Client.create(); S3Client v3Client = S3EncryptionClient.builder() - .rsaKeyPair(rsaKey) - .instructionFileConfig(InstructionFileConfig.builder() - .instructionFileClient(wrappedClient) - .enableInstructionFilePutObject(true) - .build()) - .enableDelayedAuthenticationMode(true) - .build(); + .rsaKeyPair(rsaKey) + .instructionFileConfig(InstructionFileConfig.builder() + .instructionFileClient(wrappedClient) + .enableInstructionFilePutObject(true) + .build()) + .enableDelayedAuthenticationMode(true) + .build(); CreateMultipartUploadResponse initiateResult = v3Client.createMultipartUpload(builder -> - builder.bucket(BUCKET).key(object_key).storageClass(storageClass)); + builder.bucket(BUCKET).key(object_key).storageClass(storageClass)); List partETags = new ArrayList<>(); @@ -408,57 +410,57 @@ public void testLowLevelMultipartPutWithInstructionFile() throws NoSuchAlgorithm continue; } UploadPartRequest uploadPartRequest = UploadPartRequest.builder() - .bucket(BUCKET) - .key(object_key) - .uploadId(initiateResult.uploadId()) - .partNumber(partsSent) - .build(); + .bucket(BUCKET) + .key(object_key) + .uploadId(initiateResult.uploadId()) + .partNumber(partsSent) + .build(); final InputStream partInputStream = new ByteArrayInputStream(outputStream.toByteArray()); UploadPartResponse uploadPartResult = v3Client.uploadPart(uploadPartRequest, - RequestBody.fromInputStream(partInputStream, partInputStream.available())); + RequestBody.fromInputStream(partInputStream, partInputStream.available())); partETags.add(CompletedPart.builder() - .partNumber(partsSent) - .eTag(uploadPartResult.eTag()) - .build()); + .partNumber(partsSent) + .eTag(uploadPartResult.eTag()) + .build()); outputStream.reset(); bytesSent = 0; partsSent++; } inputStream.close(); UploadPartRequest uploadPartRequest = UploadPartRequest.builder() - .bucket(BUCKET) - .key(object_key) - .uploadId(initiateResult.uploadId()) - .partNumber(partsSent) - .sdkPartType(SdkPartType.LAST) - .build(); + .bucket(BUCKET) + .key(object_key) + .uploadId(initiateResult.uploadId()) + .partNumber(partsSent) + .sdkPartType(SdkPartType.LAST) + .build(); final InputStream partInputStream = new ByteArrayInputStream(outputStream.toByteArray()); UploadPartResponse uploadPartResult = v3Client.uploadPart(uploadPartRequest, - RequestBody.fromInputStream(partInputStream, partInputStream.available())); + RequestBody.fromInputStream(partInputStream, partInputStream.available())); partETags.add(CompletedPart.builder() - .partNumber(partsSent) - .eTag(uploadPartResult.eTag()) - .build()); + .partNumber(partsSent) + .eTag(uploadPartResult.eTag()) + .build()); v3Client.completeMultipartUpload(builder -> builder - .bucket(BUCKET) - .key(object_key) - .uploadId(initiateResult.uploadId()) - .multipartUpload(partBuilder -> partBuilder.parts(partETags))); + .bucket(BUCKET) + .key(object_key) + .uploadId(initiateResult.uploadId()) + .multipartUpload(partBuilder -> partBuilder.parts(partETags))); S3Client defaultClient = S3Client.create(); ResponseBytes directInstGetResponse = defaultClient.getObjectAsBytes(builder -> builder - .bucket(BUCKET) - .key(object_key + ".instruction") - .build()); + .bucket(BUCKET) + .key(object_key + ".instruction") + .build()); assertTrue(directInstGetResponse.response().metadata().containsKey("x-amz-crypto-instr-file")); assertEquals(storageClass.toString(), directInstGetResponse.response().storageClassAsString()); ResponseInputStream getResponse = v3Client.getObject(builder -> builder - .bucket(BUCKET) - .key(object_key)); + .bucket(BUCKET) + .key(object_key)); assertTrue(IOUtils.contentEquals(objectStreamForResult, getResponse)); @@ -467,4 +469,3 @@ public void testLowLevelMultipartPutWithInstructionFile() throws NoSuchAlgorithm } } - diff --git a/src/test/java/software/amazon/encryption/s3/S3EncryptionClientMultipartUploadTest.java b/src/test/java/software/amazon/encryption/s3/S3EncryptionClientMultipartUploadTest.java index 25bbdd375..dcea6c330 100644 --- a/src/test/java/software/amazon/encryption/s3/S3EncryptionClientMultipartUploadTest.java +++ b/src/test/java/software/amazon/encryption/s3/S3EncryptionClientMultipartUploadTest.java @@ -5,8 +5,6 @@ import org.apache.commons.io.IOUtils; import org.bouncycastle.jce.provider.BouncyCastleProvider; import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.RepeatedTest; import org.junit.jupiter.api.Test; import software.amazon.awssdk.core.ResponseBytes; import software.amazon.awssdk.core.ResponseInputStream; @@ -54,7 +52,6 @@ public class S3EncryptionClientMultipartUploadTest { private static Provider PROVIDER; - private static boolean testCasePassed = false; @BeforeAll public static void setUp() throws NoSuchAlgorithmException { @@ -62,55 +59,57 @@ public static void setUp() throws NoSuchAlgorithmException { PROVIDER = Security.getProvider("BC"); } - @BeforeEach - public void resetTestCasePassedFlag() { - testCasePassed = false; - } - - @RepeatedTest(3) + @Test public void multipartPutObjectAsync() throws IOException { - if(testCasePassed) { - return; - } - final String objectKey = appendTestSuffix("multipart-put-object-async"); + int success = 0, failures = 0; + for(int i=0; i < 100; i++) { + try { + final String objectKey = appendTestSuffix("multipart-put-object-async"); - final long fileSizeLimit = 1024 * 1024 * 100; - final InputStream inputStream = new BoundedInputStream(fileSizeLimit); - final InputStream objectStreamForResult = new BoundedInputStream(fileSizeLimit); + final long fileSizeLimit = 1024 * 1024 * 100; + final InputStream inputStream = new BoundedInputStream(fileSizeLimit); + final InputStream objectStreamForResult = new BoundedInputStream(fileSizeLimit); - S3AsyncClient v3Client = S3AsyncEncryptionClient.builder() - .kmsKeyId(KMS_KEY_ID) - .enableMultipartPutObject(true) - .enableDelayedAuthenticationMode(true) - .cryptoProvider(PROVIDER) - .build(); - Map encryptionContext = new HashMap<>(); - encryptionContext.put("user-metadata-key", "user-metadata-value-v3-to-v3"); - ExecutorService singleThreadExecutor = Executors.newSingleThreadExecutor(); + S3AsyncClient v3Client = S3AsyncEncryptionClient.builder() + .kmsKeyId(KMS_KEY_ID) + .enableMultipartPutObject(true) + .enableDelayedAuthenticationMode(true) + .cryptoProvider(PROVIDER) + .build(); - CompletableFuture futurePut = v3Client.putObject(builder -> builder - .bucket(BUCKET) - .overrideConfiguration(withAdditionalConfiguration(encryptionContext)) - .key(objectKey), AsyncRequestBody.fromInputStream(inputStream, fileSizeLimit, singleThreadExecutor)); + Map encryptionContext = new HashMap<>(); + encryptionContext.put("user-metadata-key", "user-metadata-value-v3-to-v3"); - futurePut.join(); - singleThreadExecutor.shutdown(); + ExecutorService singleThreadExecutor = Executors.newSingleThreadExecutor(); - // Asserts - CompletableFuture> getFuture = v3Client.getObject(builder -> builder - .bucket(BUCKET) - .overrideConfiguration(S3EncryptionClient.withAdditionalConfiguration(encryptionContext)) - .key(objectKey), AsyncResponseTransformer.toBlockingInputStream()); - ResponseInputStream output = getFuture.join(); + CompletableFuture futurePut = v3Client.putObject(builder -> builder + .bucket(BUCKET) + .overrideConfiguration(withAdditionalConfiguration(encryptionContext)) + .key(objectKey), AsyncRequestBody.fromInputStream(inputStream, fileSizeLimit, singleThreadExecutor)); - assertTrue(IOUtils.contentEquals(objectStreamForResult, output)); + futurePut.join(); + singleThreadExecutor.shutdown(); - deleteObject(BUCKET, objectKey, v3Client); - v3Client.close(); + // Asserts + CompletableFuture> getFuture = v3Client.getObject(builder -> builder + .bucket(BUCKET) + .overrideConfiguration(S3EncryptionClient.withAdditionalConfiguration(encryptionContext)) + .key(objectKey), AsyncResponseTransformer.toBlockingInputStream()); + ResponseInputStream output = getFuture.join(); + + assertTrue(IOUtils.contentEquals(objectStreamForResult, output)); - testCasePassed = true; + deleteObject(BUCKET, objectKey, v3Client); + v3Client.close(); + + success++; + } catch (Exception e) { + failures++; + } + } + System.out.println("testMultipartPutObjectAsync: Success: "+success+" Failures: "+failures); } @Test @@ -122,11 +121,11 @@ public void multipartPutObjectAsyncLargeObjectFails() { final InputStream inputStream = new BoundedInputStream(fileSizeLimit); S3AsyncClient v3Client = S3AsyncEncryptionClient.builder() - .kmsKeyId(KMS_KEY_ID) - .enableMultipartPutObject(true) - .enableDelayedAuthenticationMode(true) - .cryptoProvider(PROVIDER) - .build(); + .kmsKeyId(KMS_KEY_ID) + .enableMultipartPutObject(true) + .enableDelayedAuthenticationMode(true) + .cryptoProvider(PROVIDER) + .build(); Map encryptionContext = new HashMap<>(); encryptionContext.put("user-metadata-key", "user-metadata-value-v3-to-v3"); @@ -134,52 +133,58 @@ public void multipartPutObjectAsyncLargeObjectFails() { ExecutorService singleThreadExecutor = Executors.newSingleThreadExecutor(); assertThrows(S3EncryptionClientException.class, () -> v3Client.putObject(builder -> builder - .bucket(BUCKET) - .overrideConfiguration(withAdditionalConfiguration(encryptionContext)) - .key(objectKey), AsyncRequestBody.fromInputStream(inputStream, fileSizeLimit, singleThreadExecutor))); + .bucket(BUCKET) + .overrideConfiguration(withAdditionalConfiguration(encryptionContext)) + .key(objectKey), AsyncRequestBody.fromInputStream(inputStream, fileSizeLimit, singleThreadExecutor))); v3Client.close(); singleThreadExecutor.shutdown(); } - @RepeatedTest(3) + @Test public void multipartPutObject() throws IOException { - if(testCasePassed) { - return; - } - final String objectKey = appendTestSuffix("multipart-put-object"); + int success = 0, failures = 0; - final long fileSizeLimit = 1024 * 1024 * 100; - final InputStream inputStream = new BoundedInputStream(fileSizeLimit); - final InputStream objectStreamForResult = new BoundedInputStream(fileSizeLimit); + for(int i=0; i < 100; i++) { + try { + final String objectKey = appendTestSuffix("multipart-put-object"); - S3Client v3Client = S3EncryptionClient.builder() - .kmsKeyId(KMS_KEY_ID) - .enableMultipartPutObject(true) - .enableDelayedAuthenticationMode(true) - .cryptoProvider(PROVIDER) - .build(); + final long fileSizeLimit = 1024 * 1024 * 100; + final InputStream inputStream = new BoundedInputStream(fileSizeLimit); + final InputStream objectStreamForResult = new BoundedInputStream(fileSizeLimit); - Map encryptionContext = new HashMap<>(); - encryptionContext.put("user-metadata-key", "user-metadata-value-v3-to-v3"); + S3Client v3Client = S3EncryptionClient.builder() + .kmsKeyId(KMS_KEY_ID) + .enableMultipartPutObject(true) + .enableDelayedAuthenticationMode(true) + .cryptoProvider(PROVIDER) + .build(); - v3Client.putObject(builder -> builder - .bucket(BUCKET) - .overrideConfiguration(withAdditionalConfiguration(encryptionContext)) - .key(objectKey), RequestBody.fromInputStream(inputStream, fileSizeLimit)); + Map encryptionContext = new HashMap<>(); + encryptionContext.put("user-metadata-key", "user-metadata-value-v3-to-v3"); - // Asserts - ResponseInputStream output = v3Client.getObject(builder -> builder - .bucket(BUCKET) - .overrideConfiguration(S3EncryptionClient.withAdditionalConfiguration(encryptionContext)) - .key(objectKey)); + v3Client.putObject(builder -> builder + .bucket(BUCKET) + .overrideConfiguration(withAdditionalConfiguration(encryptionContext)) + .key(objectKey), RequestBody.fromInputStream(inputStream, fileSizeLimit)); - assertTrue(IOUtils.contentEquals(objectStreamForResult, output)); + // Asserts + ResponseInputStream output = v3Client.getObject(builder -> builder + .bucket(BUCKET) + .overrideConfiguration(S3EncryptionClient.withAdditionalConfiguration(encryptionContext)) + .key(objectKey)); - v3Client.deleteObject(builder -> builder.bucket(BUCKET).key(objectKey)); - v3Client.close(); + assertTrue(IOUtils.contentEquals(objectStreamForResult, output)); - testCasePassed = true; + v3Client.deleteObject(builder -> builder.bucket(BUCKET).key(objectKey)); + v3Client.close(); + + success++; + } catch (Exception e) { + failures++; + } + } + System.out.println("testMultipartPutObject: Success: "+success+" Failures: "+failures); } /* @@ -291,19 +296,19 @@ public void multipartPutObjectLargeObjectFails() { final InputStream inputStream = new BoundedInputStream(fileSizeLimit); S3Client v3Client = S3EncryptionClient.builder() - .kmsKeyId(KMS_KEY_ID) - .enableMultipartPutObject(true) - .enableDelayedAuthenticationMode(true) - .cryptoProvider(PROVIDER) - .build(); + .kmsKeyId(KMS_KEY_ID) + .enableMultipartPutObject(true) + .enableDelayedAuthenticationMode(true) + .cryptoProvider(PROVIDER) + .build(); Map encryptionContext = new HashMap<>(); encryptionContext.put("user-metadata-key", "user-metadata-value-v3-to-v3"); assertThrows(S3EncryptionClientException.class, () -> v3Client.putObject(builder -> builder - .bucket(BUCKET) - .overrideConfiguration(withAdditionalConfiguration(encryptionContext)) - .key(objectKey), RequestBody.fromInputStream(inputStream, fileSizeLimit))); + .bucket(BUCKET) + .overrideConfiguration(withAdditionalConfiguration(encryptionContext)) + .key(objectKey), RequestBody.fromInputStream(inputStream, fileSizeLimit))); v3Client.close(); } @@ -320,14 +325,14 @@ public void multipartUploadV3OutputStream() throws IOException { // V3 Client S3Client v3Client = S3EncryptionClient.builder() - .kmsKeyId(KMS_KEY_ID) - .enableDelayedAuthenticationMode(true) - .cryptoProvider(PROVIDER) - .build(); + .kmsKeyId(KMS_KEY_ID) + .enableDelayedAuthenticationMode(true) + .cryptoProvider(PROVIDER) + .build(); // Create Multipart upload request to S3 CreateMultipartUploadResponse initiateResult = v3Client.createMultipartUpload(builder -> - builder.bucket(BUCKET).key(objectKey)); + builder.bucket(BUCKET).key(objectKey)); List partETags = new ArrayList<>(); @@ -345,19 +350,19 @@ public void multipartUploadV3OutputStream() throws IOException { } UploadPartRequest uploadPartRequest = UploadPartRequest.builder() - .bucket(BUCKET) - .key(objectKey) - .uploadId(initiateResult.uploadId()) - .partNumber(partsSent) - .build(); + .bucket(BUCKET) + .key(objectKey) + .uploadId(initiateResult.uploadId()) + .partNumber(partsSent) + .build(); final InputStream partInputStream = new ByteArrayInputStream(outputStream.toByteArray()); UploadPartResponse uploadPartResult = v3Client.uploadPart(uploadPartRequest, - RequestBody.fromInputStream(partInputStream, partInputStream.available())); + RequestBody.fromInputStream(partInputStream, partInputStream.available())); partETags.add(CompletedPart.builder() - .partNumber(partsSent) - .eTag(uploadPartResult.eTag()) - .build()); + .partNumber(partsSent) + .eTag(uploadPartResult.eTag()) + .build()); outputStream.reset(); bytesSent = 0; partsSent++; @@ -366,32 +371,32 @@ public void multipartUploadV3OutputStream() throws IOException { // Last Part UploadPartRequest uploadPartRequest = UploadPartRequest.builder() - .bucket(BUCKET) - .key(objectKey) - .uploadId(initiateResult.uploadId()) - .partNumber(partsSent) - .sdkPartType(SdkPartType.LAST) - .build(); + .bucket(BUCKET) + .key(objectKey) + .uploadId(initiateResult.uploadId()) + .partNumber(partsSent) + .sdkPartType(SdkPartType.LAST) + .build(); final InputStream partInputStream = new ByteArrayInputStream(outputStream.toByteArray()); UploadPartResponse uploadPartResult = v3Client.uploadPart(uploadPartRequest, - RequestBody.fromInputStream(partInputStream, partInputStream.available())); + RequestBody.fromInputStream(partInputStream, partInputStream.available())); partETags.add(CompletedPart.builder() - .partNumber(partsSent) - .eTag(uploadPartResult.eTag()) - .build()); + .partNumber(partsSent) + .eTag(uploadPartResult.eTag()) + .build()); // Complete the multipart upload. v3Client.completeMultipartUpload(builder -> builder - .bucket(BUCKET) - .key(objectKey) - .uploadId(initiateResult.uploadId()) - .multipartUpload(partBuilder -> partBuilder.parts(partETags))); + .bucket(BUCKET) + .key(objectKey) + .uploadId(initiateResult.uploadId()) + .multipartUpload(partBuilder -> partBuilder.parts(partETags))); // Asserts InputStream resultStream = v3Client.getObjectAsBytes(builder -> builder - .bucket(BUCKET) - .key(objectKey)).asInputStream(); + .bucket(BUCKET) + .key(objectKey)).asInputStream(); assertTrue(IOUtils.contentEquals(new BoundedInputStream(fileSizeLimit), resultStream)); resultStream.close(); @@ -411,14 +416,14 @@ public void multipartUploadV3OutputStreamPartSize() throws IOException { // V3 Client S3Client v3Client = S3EncryptionClient.builder() - .kmsKeyId(KMS_KEY_ID) - .enableDelayedAuthenticationMode(true) - .cryptoProvider(PROVIDER) - .build(); + .kmsKeyId(KMS_KEY_ID) + .enableDelayedAuthenticationMode(true) + .cryptoProvider(PROVIDER) + .build(); // Create Multipart upload request to S3 CreateMultipartUploadResponse initiateResult = v3Client.createMultipartUpload(builder -> - builder.bucket(BUCKET).key(objectKey)); + builder.bucket(BUCKET).key(objectKey)); List partETags = new ArrayList<>(); @@ -437,19 +442,19 @@ public void multipartUploadV3OutputStreamPartSize() throws IOException { final InputStream partInputStream = new ByteArrayInputStream(outputStream.toByteArray()); UploadPartRequest uploadPartRequest = UploadPartRequest.builder() - .bucket(BUCKET) - .key(objectKey) - .uploadId(initiateResult.uploadId()) - .partNumber(partsSent) - .contentLength((long) partInputStream.available()) - .build(); + .bucket(BUCKET) + .key(objectKey) + .uploadId(initiateResult.uploadId()) + .partNumber(partsSent) + .contentLength((long) partInputStream.available()) + .build(); UploadPartResponse uploadPartResult = v3Client.uploadPart(uploadPartRequest, - RequestBody.fromInputStream(partInputStream, partInputStream.available())); + RequestBody.fromInputStream(partInputStream, partInputStream.available())); partETags.add(CompletedPart.builder() - .partNumber(partsSent) - .eTag(uploadPartResult.eTag()) - .build()); + .partNumber(partsSent) + .eTag(uploadPartResult.eTag()) + .build()); outputStream.reset(); bytesSent = 0; partsSent++; @@ -459,32 +464,32 @@ public void multipartUploadV3OutputStreamPartSize() throws IOException { // Last Part UploadPartRequest uploadPartRequest = UploadPartRequest.builder() - .bucket(BUCKET) - .key(objectKey) - .uploadId(initiateResult.uploadId()) - .partNumber(partsSent) - .contentLength((long) partInputStream.available()) - .sdkPartType(SdkPartType.LAST) - .build(); + .bucket(BUCKET) + .key(objectKey) + .uploadId(initiateResult.uploadId()) + .partNumber(partsSent) + .contentLength((long) partInputStream.available()) + .sdkPartType(SdkPartType.LAST) + .build(); UploadPartResponse uploadPartResult = v3Client.uploadPart(uploadPartRequest, - RequestBody.fromInputStream(partInputStream, partInputStream.available())); + RequestBody.fromInputStream(partInputStream, partInputStream.available())); partETags.add(CompletedPart.builder() - .partNumber(partsSent) - .eTag(uploadPartResult.eTag()) - .build()); + .partNumber(partsSent) + .eTag(uploadPartResult.eTag()) + .build()); // Complete the multipart upload. v3Client.completeMultipartUpload(builder -> builder - .bucket(BUCKET) - .key(objectKey) - .uploadId(initiateResult.uploadId()) - .multipartUpload(partBuilder -> partBuilder.parts(partETags))); + .bucket(BUCKET) + .key(objectKey) + .uploadId(initiateResult.uploadId()) + .multipartUpload(partBuilder -> partBuilder.parts(partETags))); // Asserts ResponseBytes result = v3Client.getObjectAsBytes(builder -> builder - .bucket(BUCKET) - .key(objectKey)); + .bucket(BUCKET) + .key(objectKey)); String inputAsString = IoUtils.toUtf8String(new BoundedInputStream(fileSizeLimit)); String outputAsString = IoUtils.toUtf8String(result.asInputStream()); @@ -505,14 +510,14 @@ public void multipartUploadV3OutputStreamPartSizeMismatch() throws IOException { // V3 Client S3Client v3Client = S3EncryptionClient.builder() - .kmsKeyId(KMS_KEY_ID) - .enableDelayedAuthenticationMode(true) - .cryptoProvider(PROVIDER) - .build(); + .kmsKeyId(KMS_KEY_ID) + .enableDelayedAuthenticationMode(true) + .cryptoProvider(PROVIDER) + .build(); // Create Multipart upload request to S3 CreateMultipartUploadResponse initiateResult = v3Client.createMultipartUpload(builder -> - builder.bucket(BUCKET).key(objectKey)); + builder.bucket(BUCKET).key(objectKey)); int bytesRead, bytesSent = 0; // 10MB parts @@ -529,15 +534,15 @@ public void multipartUploadV3OutputStreamPartSizeMismatch() throws IOException { final InputStream partInputStream = new ByteArrayInputStream(outputStream.toByteArray()); UploadPartRequest uploadPartRequest = UploadPartRequest.builder() - .bucket(BUCKET) - .key(objectKey) - .uploadId(initiateResult.uploadId()) - .partNumber(partsSent) - .contentLength((long) partInputStream.available() + 1) // mismatch - .build(); + .bucket(BUCKET) + .key(objectKey) + .uploadId(initiateResult.uploadId()) + .partNumber(partsSent) + .contentLength((long) partInputStream.available() + 1) // mismatch + .build(); assertThrows(S3EncryptionClientException.class, () -> v3Client.uploadPart(uploadPartRequest, - RequestBody.fromInputStream(partInputStream, partInputStream.available()))); + RequestBody.fromInputStream(partInputStream, partInputStream.available()))); } v3Client.deleteObject(builder -> builder.bucket(BUCKET).key(objectKey)); @@ -588,4 +593,4 @@ public void multipartPutObjectWithOptions() throws IOException { v3Client.close(); } -} +} \ No newline at end of file diff --git a/src/test/java/software/amazon/encryption/s3/S3EncryptionClientStreamTest.java b/src/test/java/software/amazon/encryption/s3/S3EncryptionClientStreamTest.java index b061a9d93..9119f34b7 100644 --- a/src/test/java/software/amazon/encryption/s3/S3EncryptionClientStreamTest.java +++ b/src/test/java/software/amazon/encryption/s3/S3EncryptionClientStreamTest.java @@ -310,58 +310,64 @@ public void failsWhenBothBufferSizeAndDelayedAuthModeEnabled() { @Test public void customSetBufferSizeWithLargeObject() throws IOException { + int success = 0, failures = 0; for(int i=0; i < 100; i++) { - final String objectKey = appendTestSuffix("large-object-test-custom-buffer-size"); + try { + final String objectKey = appendTestSuffix("large-object-test-custom-buffer-size"); - Security.addProvider(new BouncyCastleProvider()); - Provider provider = Security.getProvider("BC"); + Security.addProvider(new BouncyCastleProvider()); + Provider provider = Security.getProvider("BC"); - // V3 Client with custom max buffer size 32 MiB. - S3Client v3ClientWithBuffer32MiB = S3EncryptionClient.builder() - .aesKey(AES_KEY) - .cryptoProvider(provider) - .setBufferSize(32 * 1024 * 1024) - .build(); + // V3 Client with custom max buffer size 32 MiB. + S3Client v3ClientWithBuffer32MiB = S3EncryptionClient.builder() + .aesKey(AES_KEY) + .cryptoProvider(provider) + .setBufferSize(32 * 1024 * 1024) + .build(); - // V3 Client with default buffer size (i.e. 64MiB) - // When enableDelayedAuthenticationMode is set to true, delayed authentication mode always takes priority over buffered mode. - S3Client v3ClientWithDelayedAuth = S3EncryptionClient.builder() - .aesKey(AES_KEY) - .cryptoProvider(provider) - .enableDelayedAuthenticationMode(true) - .build(); - - // Tight bound on the custom buffer size limit of 32MiB - final long fileSizeExceedingDefaultLimit = 1024 * 1024 * 32 + 1; - - final InputStream largeObjectStream = new MarkResetBoundedZerosInputStream(fileSizeExceedingDefaultLimit); - v3ClientWithBuffer32MiB.putObject(PutObjectRequest.builder() - .bucket(BUCKET) - .key(objectKey) - .build(), RequestBody.fromInputStream(largeObjectStream, fileSizeExceedingDefaultLimit)); - - largeObjectStream.close(); - - // Object is larger than Buffer, so getObject fails - assertThrows(S3EncryptionClientException.class, () -> v3ClientWithBuffer32MiB.getObjectAsBytes(builder -> builder - .bucket(BUCKET) - .key(objectKey))); - - // You have to either enable the delayed auth mode or increase max buffer size (but in allowed bounds) - ResponseInputStream response = v3ClientWithDelayedAuth.getObject(builder -> builder - .bucket(BUCKET) - .key(objectKey)); - - // Create a new ByteArrayInputStream for comparison - InputStream expectedStream = new MarkResetBoundedZerosInputStream(fileSizeExceedingDefaultLimit); - assertTrue(IOUtils.contentEquals(expectedStream, response)); - response.close(); - - // Cleanup - deleteObject(BUCKET, objectKey, v3ClientWithBuffer32MiB); - v3ClientWithBuffer32MiB.close(); - v3ClientWithDelayedAuth.close(); + // V3 Client with default buffer size (i.e. 64MiB) + // When enableDelayedAuthenticationMode is set to true, delayed authentication mode always takes priority over buffered mode. + S3Client v3ClientWithDelayedAuth = S3EncryptionClient.builder() + .aesKey(AES_KEY) + .cryptoProvider(provider) + .enableDelayedAuthenticationMode(true) + .build(); + + // Tight bound on the custom buffer size limit of 32MiB + final long fileSizeExceedingDefaultLimit = 1024 * 1024 * 32 + 1; + final InputStream largeObjectStream = new BoundedInputStream(fileSizeExceedingDefaultLimit); + v3ClientWithBuffer32MiB.putObject(PutObjectRequest.builder() + .bucket(BUCKET) + .key(objectKey) + .build(), RequestBody.fromInputStream(largeObjectStream, fileSizeExceedingDefaultLimit)); + + largeObjectStream.close(); + + // Object is larger than Buffer, so getObject fails + assertThrows(S3EncryptionClientException.class, () -> v3ClientWithBuffer32MiB.getObjectAsBytes(builder -> builder + .bucket(BUCKET) + .key(objectKey))); + + // You have to either enable the delayed auth mode or increase max buffer size (but in allowed bounds) + ResponseInputStream response = v3ClientWithDelayedAuth.getObject(builder -> builder + .bucket(BUCKET) + .key(objectKey)); + + + assertTrue(IOUtils.contentEquals(new BoundedInputStream(fileSizeExceedingDefaultLimit), response)); + response.close(); + + // Cleanup + deleteObject(BUCKET, objectKey, v3ClientWithBuffer32MiB); + v3ClientWithBuffer32MiB.close(); + v3ClientWithDelayedAuth.close(); + + success++; + } catch (Exception e) { + failures++; + } } + System.out.println("Success: "+success+" Failures: "+failures); } @Test @@ -391,8 +397,7 @@ public void customSetBufferSizeWithLargeObjectAsyncClient() throws IOException { // Tight bound on the custom buffer size limit of 32MiB final long fileSizeExceedingDefaultLimit = 1024 * 1024 * 32 + 1; - final InputStream largeObjectStream = new MarkResetBoundedZerosInputStream(fileSizeExceedingDefaultLimit); - + final InputStream largeObjectStream = new BoundedInputStream(fileSizeExceedingDefaultLimit); ExecutorService singleThreadExecutor = Executors.newSingleThreadExecutor(); CompletableFuture futurePut = v3ClientWithBuffer32MiB.putObject(PutObjectRequest.builder() .bucket(BUCKET) @@ -419,9 +424,7 @@ public void customSetBufferSizeWithLargeObjectAsyncClient() throws IOException { .key(objectKey), AsyncResponseTransformer.toBlockingInputStream()); ResponseInputStream output = futureGet.join(); - InputStream expectedStream = new MarkResetBoundedZerosInputStream(fileSizeExceedingDefaultLimit); - assertTrue(IOUtils.contentEquals(expectedStream, output)); - + assertTrue(IOUtils.contentEquals(new BoundedInputStream(fileSizeExceedingDefaultLimit), output)); output.close(); // Cleanup @@ -455,9 +458,7 @@ public void delayedAuthModeWithLargeObject() throws IOException { // Tight bound on the default limit of 64MiB final long fileSizeExceedingDefaultLimit = 1024 * 1024 * 64 + 1; - final InputStream largeObjectStream = new BufferedInputStream( - new BoundedInputStream(fileSizeExceedingDefaultLimit) - ); + final InputStream largeObjectStream = new BoundedInputStream(fileSizeExceedingDefaultLimit); v3Client.putObject(PutObjectRequest.builder() .bucket(BUCKET) .key(objectKey) diff --git a/src/test/java/software/amazon/encryption/s3/examples/MultipartUploadExampleTest.java b/src/test/java/software/amazon/encryption/s3/examples/MultipartUploadExampleTest.java index ee87c39d7..0e03a0f80 100644 --- a/src/test/java/software/amazon/encryption/s3/examples/MultipartUploadExampleTest.java +++ b/src/test/java/software/amazon/encryption/s3/examples/MultipartUploadExampleTest.java @@ -6,21 +6,21 @@ import static org.junit.jupiter.api.Assertions.fail; public class MultipartUploadExampleTest { - private static boolean testCasePassed = false; @Test public void testMultipartUploadExamples() { - if(testCasePassed) { - return; - } - - final String bucket = S3EncryptionClientTestResources.BUCKET; - try { - MultipartUploadExample.main(new String[]{bucket}); - testCasePassed = true; - } catch (Throwable exception) { - exception.printStackTrace(); - fail("Multipart Example Test Failed!!", exception); + int success = 0, failures = 0; + for(int i=0; i < 100; i++) { + final String bucket = S3EncryptionClientTestResources.BUCKET; + try { + MultipartUploadExample.main(new String[]{bucket}); + success++; + } catch (Throwable exception) { + exception.printStackTrace(); + fail("Multipart Example Test Failed!!", exception); + failures++; + } } + System.out.println("testMultipartUploadExamples: Success: "+success+" Failures: "+failures); } }