From 9abfc30074920521188cb516dc0198d0079ec978 Mon Sep 17 00:00:00 2001 From: texastony <5892063+texastony@users.noreply.github.com> Date: Fri, 21 Jun 2024 14:14:26 -0700 Subject: [PATCH 1/7] refactor(tests): consolidate common test variables --- .../S3EncryptionClientCompatibilityTest.java | 17 ++-- .../S3EncryptionClientTestResources.java | 80 +++++++++++++++++++ src/test/resources/RSAPrivateKey.pem | 28 +++++++ src/test/resources/RSAPublicKey.pem | 9 +++ 4 files changed, 126 insertions(+), 8 deletions(-) create mode 100644 src/test/resources/RSAPrivateKey.pem create mode 100644 src/test/resources/RSAPublicKey.pem diff --git a/src/test/java/software/amazon/encryption/s3/S3EncryptionClientCompatibilityTest.java b/src/test/java/software/amazon/encryption/s3/S3EncryptionClientCompatibilityTest.java index 3e8fa78fe..5dbc23f3d 100644 --- a/src/test/java/software/amazon/encryption/s3/S3EncryptionClientCompatibilityTest.java +++ b/src/test/java/software/amazon/encryption/s3/S3EncryptionClientCompatibilityTest.java @@ -28,6 +28,7 @@ import software.amazon.awssdk.services.s3.model.GetObjectRequest; import software.amazon.awssdk.services.s3.model.GetObjectResponse; import software.amazon.awssdk.services.s3.model.PutObjectRequest; +import software.amazon.encryption.s3.utils.S3EncryptionClientTestResources; import javax.crypto.KeyGenerator; import javax.crypto.SecretKey; @@ -37,6 +38,7 @@ import java.security.KeyPair; import java.security.KeyPairGenerator; import java.security.NoSuchAlgorithmException; +import java.security.spec.InvalidKeySpecException; import java.util.HashMap; import java.util.Map; @@ -45,29 +47,28 @@ import static software.amazon.encryption.s3.S3EncryptionClient.withAdditionalConfiguration; import static software.amazon.encryption.s3.utils.S3EncryptionClientTestResources.appendTestSuffix; import static software.amazon.encryption.s3.utils.S3EncryptionClientTestResources.deleteObject; +import static software.amazon.encryption.s3.utils.S3EncryptionClientTestResources.BUCKET; +import static software.amazon.encryption.s3.utils.S3EncryptionClientTestResources.KMS_KEY_ID; /** * This class is an integration test for verifying compatibility of ciphertexts * between V1, V2, and V3 clients under various conditions. */ public class S3EncryptionClientCompatibilityTest { - - private static final String BUCKET = System.getenv("AWS_S3EC_TEST_BUCKET"); - private static final String KMS_KEY_ID = System.getenv("AWS_S3EC_TEST_KMS_KEY_ID"); - private static final Region KMS_REGION = Region.getRegion(Regions.fromName(System.getenv("AWS_REGION"))); + // SDK V1 Region + private static final Region KMS_REGION = Region.getRegion( + Regions.fromName(S3EncryptionClientTestResources.KMS_REGION.toString())); private static SecretKey AES_KEY; private static KeyPair RSA_KEY_PAIR; @BeforeAll - public static void setUp() throws NoSuchAlgorithmException { + public static void setUp() throws NoSuchAlgorithmException, IOException, InvalidKeySpecException { KeyGenerator keyGen = KeyGenerator.getInstance("AES"); keyGen.init(256); AES_KEY = keyGen.generateKey(); - KeyPairGenerator keyPairGen = KeyPairGenerator.getInstance("RSA"); - keyPairGen.initialize(2048); - RSA_KEY_PAIR = keyPairGen.generateKeyPair(); + RSA_KEY_PAIR = S3EncryptionClientTestResources.getRSAKeyPair(); } @Test diff --git a/src/test/java/software/amazon/encryption/s3/utils/S3EncryptionClientTestResources.java b/src/test/java/software/amazon/encryption/s3/utils/S3EncryptionClientTestResources.java index 0aa498380..73453611e 100644 --- a/src/test/java/software/amazon/encryption/s3/utils/S3EncryptionClientTestResources.java +++ b/src/test/java/software/amazon/encryption/s3/utils/S3EncryptionClientTestResources.java @@ -2,12 +2,30 @@ // SPDX-License-Identifier: Apache-2.0 package software.amazon.encryption.s3.utils; +import org.bouncycastle.util.io.pem.PemObject; +import org.bouncycastle.util.io.pem.PemReader; +import org.bouncycastle.util.io.pem.PemWriter; import org.joda.time.DateTime; import org.joda.time.format.DateTimeFormat; +import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider; +import software.amazon.awssdk.auth.credentials.DefaultCredentialsProvider; +import software.amazon.awssdk.http.SdkHttpClient; +import software.amazon.awssdk.http.apache.ApacheHttpClient; +import software.amazon.awssdk.regions.Region; import software.amazon.awssdk.services.s3.S3AsyncClient; import software.amazon.awssdk.services.s3.S3Client; import software.amazon.awssdk.services.s3.model.DeleteObjectResponse; +import java.io.*; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.security.*; +import java.security.interfaces.RSAPrivateKey; +import java.security.interfaces.RSAPublicKey; +import java.security.spec.InvalidKeySpecException; +import java.security.spec.PKCS8EncodedKeySpec; +import java.security.spec.X509EncodedKeySpec; import java.util.concurrent.CompletableFuture; /** @@ -15,10 +33,13 @@ */ public class S3EncryptionClientTestResources { + public static final AwsCredentialsProvider CREDENTIALS = DefaultCredentialsProvider.create(); + public static final SdkHttpClient HTTP_CLIENT = ApacheHttpClient.create(); public static final String BUCKET = System.getenv("AWS_S3EC_TEST_BUCKET"); public static final String KMS_KEY_ID = System.getenv("AWS_S3EC_TEST_KMS_KEY_ID"); // This alias must point to the same key as KMS_KEY_ID public static final String KMS_KEY_ALIAS = System.getenv("AWS_S3EC_TEST_KMS_KEY_ALIAS"); + public static final Region KMS_REGION = Region.of(System.getenv("AWS_REGION")); /** * For a given string, append a suffix to distinguish it from @@ -57,4 +78,63 @@ public static void deleteObject(final String bucket, final String objectKey, fin // Ensure completion before return response.join(); } + + + /** + * @return If an RSA KeyPair already exists in Test Resources, load and return that.

+ * Otherwise, generate a new key pair, persist that to Resources, and return it.

+ * Assumes working directory is root of the git repo. + */ + public static KeyPair getRSAKeyPair() throws IOException, NoSuchAlgorithmException, InvalidKeySpecException { + Path resourceDirectory = Paths.get("src","test","resources"); + if (resourceDirectory.resolve("RSAPrivateKey.pem").toFile().exists() && resourceDirectory.resolve("RSAPublicKey.pem").toFile().exists()) { + return readKeyPairFromTestResourcesFile(resourceDirectory); + } + KeyPair keyPair = generateKeyPair(2048); + writeKeyPairToTestResourcesFile(keyPair, resourceDirectory); + return keyPair; + } + + public static KeyPair generateKeyPair(final int keySize) { + if (!(keySize == 2048 || keySize == 4096)) throw new IllegalArgumentException("Only 2048 or 4096 are valid key sizes."); + KeyPairGenerator rsaGen; + try { + rsaGen = KeyPairGenerator.getInstance("RSA"); + } catch (NoSuchAlgorithmException e) { + throw new RuntimeException("No such algorithm", e); + } + rsaGen.initialize(keySize, new SecureRandom()); + return rsaGen.generateKeyPair(); + } + + private static void writePEMFile(Key key, String description, Path filePath) throws IOException { + final PemObject pemObject = new PemObject(description, key.getEncoded()); + try (PemWriter pemWriter = new PemWriter(new OutputStreamWriter(Files.newOutputStream(filePath)))) { + pemWriter.writeObject(pemObject); + } + } + + private static PemObject readPEMFile(Path filePath) throws IOException { + try (PemReader pemReader = new PemReader(new InputStreamReader(Files.newInputStream(filePath)))) { + return pemReader.readPemObject(); + } + } + + private static void writeKeyPairToTestResourcesFile(final KeyPair keyPair, Path resourceDirectory) throws IOException { + RSAPrivateKey privateKey = (RSAPrivateKey) keyPair.getPrivate(); + writePEMFile(privateKey, "RSA PRIVATE KEY", resourceDirectory.resolve("RSAPrivateKey.pem")); + RSAPublicKey publicKey = (RSAPublicKey) keyPair.getPublic(); + writePEMFile(publicKey, "RSA PUBLIC KEY", resourceDirectory.resolve("RSAPublicKey.pem")); + } + + private static KeyPair readKeyPairFromTestResourcesFile(Path resourceDirectory) throws IOException, NoSuchAlgorithmException, InvalidKeySpecException { + final KeyFactory factory = KeyFactory.getInstance("RSA"); + byte[] privateKeyContent = readPEMFile(resourceDirectory.resolve("RSAPrivateKey.pem")).getContent(); + PKCS8EncodedKeySpec privateKeySpec = new PKCS8EncodedKeySpec(privateKeyContent); + final PrivateKey privateKey = factory.generatePrivate(privateKeySpec); + byte[] publicKeyContent = readPEMFile(resourceDirectory.resolve("RSAPublicKey.pem")).getContent(); + X509EncodedKeySpec publicKeySpec = new X509EncodedKeySpec(publicKeyContent); + final PublicKey publicKey = factory.generatePublic(publicKeySpec); + return new KeyPair(publicKey, privateKey); + } } diff --git a/src/test/resources/RSAPrivateKey.pem b/src/test/resources/RSAPrivateKey.pem new file mode 100644 index 000000000..5019447a9 --- /dev/null +++ b/src/test/resources/RSAPrivateKey.pem @@ -0,0 +1,28 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQDDhiKZterVY9+h +igrZMn8gzfcNEXhqDoCynjWewiQriG3P4DuasePNVjQSIk7tp1/sn6zSeIm/UlAv +8Mtx+dh2MG9l6jyFiAm8T7heU9CgdSc5Dp5ZJ2QrSwXoQPBnrHTSomxB3sg75Uhn +hhv1MfVceFxKPUJTXdQ6Mb6hy14RGfx06E3Ugffvy81rNf4e9A7uUbktVocYUqvO +aiv8JzGYnDXdlhtfZ/DYMnJMH73AFJJ+7+XAfUMU32nvhkcGhiKj6auJXB2makJ3 +FUtwrNXYMc976WrH9Kk2iu33rL7PV6n9+X6xI/WGdf9X0OCORzbk6ih1CEIU0J+M +KIkRGAirAgMBAAECggEAB9u6REdFevIaqNltejFHXsAob8QF/O08SvGE4i6XWZCQ +KUyv2JXRvAz85sWuOmsBtfbs8UCa+K+MPYEGDDyocIed0pDJgexnx8PEezYPKoPK +4cYuoxKsOfk38Y+6mdAameShSTx0+8NJV6/SK9aoL+E+hFVV9xfMUdJyAPq1eyZo +PTxnUvV4INehN/rQL3X+00XjSzYEUo5IjJldqyvEVOAmcgxUXekgHUIcOa13uTMi +X9pAcvVH9LV81AFe/s8r7Ob15GWfe8Vyny3hDbj1EtG40vKK512JL2EKJy25DgsR +sbCWDLLv/2LpOp8mt4X3bxkR9WuKWW4o30ef1OZdkQKBgQDfoy2t1ifBVsmPCEKQ +3ChAW+hcdRxtD2+yP6L4DZlrIi12UdgojwggUKe81AV/C5NDTr4h2cqzA5EQNT+V +CdpNND1zcRdbNqqZiOaV1FCaORijfmimmnfBUNKrpKP5E38lV6PfAlF1i45QFryn +0kuAAA/FlCF+IBhy8C3ksncAZwKBgQDf0XjepXFu8B1zfZP4N4EjASO3+NQY0BGE +19+rOs5br1bnw3Q82Y89vAC/mPuXRy32ENaSN2RVK8vFds54bsG6NzdRnwWY5deH +0x1jCZn3/6DicviFA0O0TCMFErTG7DwRHRL58ftyV4lZk9kxq8h/x9deKm0a8cEW +HZVC7wj7HQKBgF63sQgYVNwpEtMWj4LlC9M+WeqW20RBrnATTcW7lMfwQMsFHQUI +l0uAfZqXPgCx+VwfhJ23rYcmMpFnzBcmhiP+xSwYsOi7/YNrnSXGN6EqH4pXZqFx +eNkSjzeNUrmSjV5WgRxZ0gBz7AF1r89wXPPIkuV+uLS/iTtdCEL9ZzNvAoGBALY3 +6Fv7/fn/6zpXhtyS88P37YieQK9i1qB80FCrs83ZVruh2UShK4lrQoC6oDptbPHk +i4zHJBxjZ6cALuDF61scESGWggwVNAAU1NwIuR27NNSoHcTM/5YOVoSO0jcRpWWZ +chWj+L8CnYQcZruVy8qcfK7hg6poIHdM5nRz/6/RAoGARtGMAM3CoS8sHB3HYrzZ +gfzKImHSCADCHz8eo+17SXLf4M4v769M8luicd1a0vaCFrFa5vySe3FiizXtqZa/ +cPpCUxxX4hOnQJ8Mki875JajBgamd60ZJE35ZvlyX8obq4YrSLm2WUQ9aqaHT3dh +qL/371EPip3eVdvNAyqjwBc= +-----END RSA PRIVATE KEY----- diff --git a/src/test/resources/RSAPublicKey.pem b/src/test/resources/RSAPublicKey.pem new file mode 100644 index 000000000..055278cec --- /dev/null +++ b/src/test/resources/RSAPublicKey.pem @@ -0,0 +1,9 @@ +-----BEGIN RSA PUBLIC KEY----- +MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAw4YimbXq1WPfoYoK2TJ/ +IM33DRF4ag6Asp41nsIkK4htz+A7mrHjzVY0EiJO7adf7J+s0niJv1JQL/DLcfnY +djBvZeo8hYgJvE+4XlPQoHUnOQ6eWSdkK0sF6EDwZ6x00qJsQd7IO+VIZ4Yb9TH1 +XHhcSj1CU13UOjG+octeERn8dOhN1IH378vNazX+HvQO7lG5LVaHGFKrzmor/Ccx +mJw13ZYbX2fw2DJyTB+9wBSSfu/lwH1DFN9p74ZHBoYio+mriVwdpmpCdxVLcKzV +2DHPe+lqx/SpNort96y+z1ep/fl+sSP1hnX/V9Dgjkc25OoodQhCFNCfjCiJERgI +qwIDAQAB +-----END RSA PUBLIC KEY----- From 6facc438caa9b7570e3dc39c5295d05632fb8078 Mon Sep 17 00:00:00 2001 From: texastony <5892063+texastony@users.noreply.github.com> Date: Fri, 21 Jun 2024 15:44:12 -0700 Subject: [PATCH 2/7] test(#300): determine streaming range issue b/w regular & encryption --- pom.xml | 6 +- .../s3/examples/TestEndOfStreamBehavior.java | 136 ++++++++++++++++++ 2 files changed, 139 insertions(+), 3 deletions(-) create mode 100644 src/test/java/software/amazon/encryption/s3/examples/TestEndOfStreamBehavior.java diff --git a/pom.xml b/pom.xml index f11c0f1b5..888a07ac3 100644 --- a/pom.xml +++ b/pom.xml @@ -56,7 +56,7 @@ software.amazon.awssdk bom - 2.20.38 + 2.26.7 true pom import @@ -68,13 +68,13 @@ software.amazon.awssdk s3 - 2.20.38 + 2.26.7 software.amazon.awssdk kms - 2.20.38 + 2.26.7 true diff --git a/src/test/java/software/amazon/encryption/s3/examples/TestEndOfStreamBehavior.java b/src/test/java/software/amazon/encryption/s3/examples/TestEndOfStreamBehavior.java new file mode 100644 index 000000000..a3155a566 --- /dev/null +++ b/src/test/java/software/amazon/encryption/s3/examples/TestEndOfStreamBehavior.java @@ -0,0 +1,136 @@ +package software.amazon.encryption.s3.examples; + +import java.io.InputStream; +import java.nio.ByteBuffer; +import java.security.KeyPair; +import java.util.stream.Stream; + +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.MethodSource; +import software.amazon.awssdk.core.sync.RequestBody; +import software.amazon.awssdk.http.async.SdkAsyncHttpClient; +import software.amazon.awssdk.http.nio.netty.NettyNioAsyncHttpClient; +import software.amazon.awssdk.regions.Region; +import software.amazon.awssdk.services.s3.S3AsyncClient; +import software.amazon.awssdk.services.s3.S3Client; +import software.amazon.awssdk.services.s3.model.DeleteObjectRequest; +import software.amazon.awssdk.services.s3.model.GetObjectRequest; +import software.amazon.awssdk.services.s3.model.PutObjectRequest; +import software.amazon.encryption.s3.S3EncryptionClient; +import software.amazon.encryption.s3.utils.S3EncryptionClientTestResources; + +import static software.amazon.encryption.s3.utils.S3EncryptionClientTestResources.*; + +public class TestEndOfStreamBehavior { + private static final Region DEFAULT_REGION = KMS_REGION; + private static final String KEY = "GHI-300.txt"; + @SuppressWarnings("SpellCheckingInspection") + private static final byte[] CONTENT = new String(new char[4]) + .replace("\0", "abcdefghijklmnopqrstuvwxyz0123456789") + .getBytes(); + /** The encryption key to use in client-side encryption tests. */ + protected static final KeyPair KEY_PAIR; + + static { + try { + KEY_PAIR = S3EncryptionClientTestResources.getRSAKeyPair(); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + static Stream clientProvider() { + return Stream.of( + getClient(DEFAULT_REGION), + getEncryptionClient(KEY_PAIR, DEFAULT_REGION)); + } + + @ParameterizedTest + @MethodSource("clientProvider") + void testEndOfStreamBehavior(final S3Client client) throws Exception { + // Delete the data if it exists + final DeleteObjectRequest deleteRequest = DeleteObjectRequest.builder() + .bucket(BUCKET) + .key(KEY) + .build(); + + client.deleteObject(deleteRequest); + + // Upload the data + final PutObjectRequest uploadRequest = + PutObjectRequest.builder().bucket(BUCKET).key(KEY).build(); + client.putObject(uploadRequest, RequestBody.fromBytes(CONTENT)); + // wait 5 seconds for the data to be uploaded + Thread.sleep(5000); + + // Actual test + final GetObjectRequest downloadRequest = + GetObjectRequest.builder() + .bucket(BUCKET) + .key(KEY) + .range("bytes=0-15") + .build(); + + final InputStream stream = client.getObject(downloadRequest); + + // Buffer capacity matters !!! + // Behavior difference when the capacity is same as the content length (i.e. 16) of the ranged query + final ByteBuffer buffer = ByteBuffer.allocate(16); + final byte[] underlyingBuffer = buffer.array(); + final int capacity = buffer.capacity(); + + final int END_OF_STREAM = -1; + int byteRead = 0; + int startPosition = 0; + while (byteRead != END_OF_STREAM) { + int lenToRead = capacity - startPosition; + System.out.println("Start position: " + startPosition + " Length to read: " + lenToRead); + // @NathanEckert , about https://github.com/aws/amazon-s3-encryption-client-java/issues/300 + // Crypto Tools SOMETIMES got an Assertion Error from + // https://github.com/aws/aws-sdk-java-v2/blob/2.20.38/utils/src/main/java/software/amazon/awssdk/utils/async/InputStreamSubscriber.java#L110 + // when using the Encryption Client. + // If we bump our Java SDK dependencies to the latest, which today is 2.26.7, + // than we never get the Assertion Error. + // Here is the PR that changes InputStreamSubscriber b/w 2.20.38 and 2.26.7: + // https://github.com/aws/aws-sdk-java-v2/pull/5201 + // This makes us suspect that something else is going wrong. + // Otherwise, we cannot detect a difference in behavior between + // the S3EC V3 Client and the S3 V2 Client with respect to this code. + byteRead = stream.read(underlyingBuffer, startPosition, lenToRead); + System.out.println("Read " + byteRead + " bytes"); + startPosition += byteRead; + if (byteRead == 0) { + // Crypto Tools cannot get this case to ever occur. + System.out.println("Looping indefinitely with an encryption client, as startPosition is not increasing"); + break; + } + } + } + + public static S3Client getEncryptionClient(final KeyPair keyPair, final Region region) { + return S3EncryptionClient.builder() + .rsaKeyPair(keyPair) + .enableLegacyUnauthenticatedModes(true) + .wrappedClient(getClient(region)) + .wrappedAsyncClient(getAsyncClient(region)) + .build(); + } + + public static S3Client getClient(final Region region) { + return S3Client.builder() + .region(region) + .credentialsProvider(CREDENTIALS) + .httpClient(HTTP_CLIENT) + .build(); + } + + public static S3AsyncClient getAsyncClient(final Region region) { + final SdkAsyncHttpClient nettyHttpClient = + NettyNioAsyncHttpClient.builder().maxConcurrency(100).build(); + return S3AsyncClient.builder() + .region(region) + .credentialsProvider(CREDENTIALS) + .httpClient(nettyHttpClient) + .build(); + } +} From f7467dbd45c96d9f31b5ecc5c91cacf56180bd56 Mon Sep 17 00:00:00 2001 From: texastony <5892063+texastony@users.noreply.github.com> Date: Mon, 24 Jun 2024 09:23:03 -0700 Subject: [PATCH 3/7] test: always generate unique RSA keypair for compatability tests --- .../encryption/s3/S3EncryptionClientCompatibilityTest.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/test/java/software/amazon/encryption/s3/S3EncryptionClientCompatibilityTest.java b/src/test/java/software/amazon/encryption/s3/S3EncryptionClientCompatibilityTest.java index 5dbc23f3d..328837eb7 100644 --- a/src/test/java/software/amazon/encryption/s3/S3EncryptionClientCompatibilityTest.java +++ b/src/test/java/software/amazon/encryption/s3/S3EncryptionClientCompatibilityTest.java @@ -63,12 +63,14 @@ public class S3EncryptionClientCompatibilityTest { private static KeyPair RSA_KEY_PAIR; @BeforeAll - public static void setUp() throws NoSuchAlgorithmException, IOException, InvalidKeySpecException { + public static void setUp() throws NoSuchAlgorithmException { KeyGenerator keyGen = KeyGenerator.getInstance("AES"); keyGen.init(256); AES_KEY = keyGen.generateKey(); - RSA_KEY_PAIR = S3EncryptionClientTestResources.getRSAKeyPair(); + KeyPairGenerator keyPairGen = KeyPairGenerator.getInstance("RSA"); + keyPairGen.initialize(2048); + RSA_KEY_PAIR = keyPairGen.generateKeyPair(); } @Test From 236147b6e21725a1431d4f61134848130e9e92f3 Mon Sep 17 00:00:00 2001 From: texastony <5892063+texastony@users.noreply.github.com> Date: Mon, 24 Jun 2024 09:46:59 -0700 Subject: [PATCH 4/7] fix: SDK V2 update needs updated CRT --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 97b12e2b7..8f2b224c6 100644 --- a/pom.xml +++ b/pom.xml @@ -82,7 +82,7 @@ software.amazon.awssdk.crt aws-crt - 0.21.9 + 0.29.25 true From 9eca136e17e37a6d02c69896ea8d74b41cada7fa Mon Sep 17 00:00:00 2001 From: texastony <5892063+texastony@users.noreply.github.com> Date: Mon, 24 Jun 2024 10:09:37 -0700 Subject: [PATCH 5/7] chore: consolidate AWS SDK V2 version to one variable Replace `aws-crt` dependency with `aws-crt-client`. --- pom.xml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/pom.xml b/pom.xml index 8f2b224c6..7600f1372 100644 --- a/pom.xml +++ b/pom.xml @@ -41,6 +41,7 @@ 8 8 UTF-8 + 2.26.7 @@ -56,7 +57,7 @@ software.amazon.awssdk bom - 2.26.7 + ${aws.java.sdk.version} true pom import @@ -68,21 +69,20 @@ software.amazon.awssdk s3 - 2.26.7 + ${aws.java.sdk.version} software.amazon.awssdk kms - 2.26.7 + ${aws.java.sdk.version} true - - software.amazon.awssdk.crt - aws-crt - 0.29.25 + software.amazon.awssdk + aws-crt-client + ${aws.java.sdk.version} true From cebdb6aebffd0a1ea084e494a9bb7e47312fb7dd Mon Sep 17 00:00:00 2001 From: texastony <5892063+texastony@users.noreply.github.com> Date: Mon, 24 Jun 2024 10:10:02 -0700 Subject: [PATCH 6/7] chore: use consistent RSA Key Pair --- .../encryption/s3/S3EncryptionClientCompatibilityTest.java | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/test/java/software/amazon/encryption/s3/S3EncryptionClientCompatibilityTest.java b/src/test/java/software/amazon/encryption/s3/S3EncryptionClientCompatibilityTest.java index 328837eb7..5dbc23f3d 100644 --- a/src/test/java/software/amazon/encryption/s3/S3EncryptionClientCompatibilityTest.java +++ b/src/test/java/software/amazon/encryption/s3/S3EncryptionClientCompatibilityTest.java @@ -63,14 +63,12 @@ public class S3EncryptionClientCompatibilityTest { private static KeyPair RSA_KEY_PAIR; @BeforeAll - public static void setUp() throws NoSuchAlgorithmException { + public static void setUp() throws NoSuchAlgorithmException, IOException, InvalidKeySpecException { KeyGenerator keyGen = KeyGenerator.getInstance("AES"); keyGen.init(256); AES_KEY = keyGen.generateKey(); - KeyPairGenerator keyPairGen = KeyPairGenerator.getInstance("RSA"); - keyPairGen.initialize(2048); - RSA_KEY_PAIR = keyPairGen.generateKeyPair(); + RSA_KEY_PAIR = S3EncryptionClientTestResources.getRSAKeyPair(); } @Test From 56e28c14d43cbec624bc2eea791cfe7653adfda8 Mon Sep 17 00:00:00 2001 From: texastony <5892063+texastony@users.noreply.github.com> Date: Mon, 24 Jun 2024 12:29:36 -0700 Subject: [PATCH 7/7] wip(#300): @NathanEckert is correct The modern S3EC and SDK V2 treat reading 0 bytes from a stream differently in at least one case: If the Stream has no more content. --- .github/workflows/ghi_300.yml | 49 +++++++++++++++++++ .../s3/examples/TestEndOfStreamBehavior.java | 19 ++----- 2 files changed, 54 insertions(+), 14 deletions(-) create mode 100644 .github/workflows/ghi_300.yml diff --git a/.github/workflows/ghi_300.yml b/.github/workflows/ghi_300.yml new file mode 100644 index 000000000..c309a528f --- /dev/null +++ b/.github/workflows/ghi_300.yml @@ -0,0 +1,49 @@ +name: ghi_300.yml +on: + push: + branches: + - tony/refactor-tests + - 'ghi-300/**' + +jobs: + Build: + runs-on: ubuntu-latest + permissions: + id-token: write + contents: read + + steps: + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v2 + with: + role-to-assume: arn:aws:iam::${{ secrets.CI_AWS_ACCOUNT_ID }}:role/service-role/${{ vars.CI_AWS_ROLE }} + role-session-name: S3EC-Github-CI-Tests + aws-region: ${{ vars.CI_AWS_REGION }} + + - name: Checkout Code + uses: actions/checkout@v3 + + # TODO: Add OpenJDK + # OpenJDK would require a different action than setup-java, so setup is more involved. + + - name: Setup JDK + uses: actions/setup-java@v3 + with: + distribution: corretto + java-version: 8 + cache: 'maven' + + - name: Compile + run: | + mvn --batch-mode -no-transfer-progress clean compile + mvn --batch-mode -no-transfer-progress test-compile + shell: bash + + - name: Test + run: | + export AWS_S3EC_TEST_BUCKET=${{ vars.CI_S3_BUCKET }} + export AWS_S3EC_TEST_KMS_KEY_ID=arn:aws:kms:${{ vars.CI_AWS_REGION }}:${{ secrets.CI_AWS_ACCOUNT_ID }}:key/${{ vars.CI_KMS_KEY_ID }} + export AWS_S3EC_TEST_KMS_KEY_ALIAS=arn:aws:kms:${{ vars.CI_AWS_REGION }}:${{ secrets.CI_AWS_ACCOUNT_ID }}:alias/${{ vars.CI_KMS_KEY_ALIAS }} + export AWS_REGION=${{ vars.CI_AWS_REGION }} + mvn -B -ntp -DskipCompile -Dtest=software.amazon.encryption.s3.examples.TestEndOfStreamBehavior test + shell: bash diff --git a/src/test/java/software/amazon/encryption/s3/examples/TestEndOfStreamBehavior.java b/src/test/java/software/amazon/encryption/s3/examples/TestEndOfStreamBehavior.java index a3155a566..a23de87e8 100644 --- a/src/test/java/software/amazon/encryption/s3/examples/TestEndOfStreamBehavior.java +++ b/src/test/java/software/amazon/encryption/s3/examples/TestEndOfStreamBehavior.java @@ -85,24 +85,15 @@ void testEndOfStreamBehavior(final S3Client client) throws Exception { while (byteRead != END_OF_STREAM) { int lenToRead = capacity - startPosition; System.out.println("Start position: " + startPosition + " Length to read: " + lenToRead); - // @NathanEckert , about https://github.com/aws/amazon-s3-encryption-client-java/issues/300 - // Crypto Tools SOMETIMES got an Assertion Error from - // https://github.com/aws/aws-sdk-java-v2/blob/2.20.38/utils/src/main/java/software/amazon/awssdk/utils/async/InputStreamSubscriber.java#L110 - // when using the Encryption Client. - // If we bump our Java SDK dependencies to the latest, which today is 2.26.7, - // than we never get the Assertion Error. - // Here is the PR that changes InputStreamSubscriber b/w 2.20.38 and 2.26.7: - // https://github.com/aws/aws-sdk-java-v2/pull/5201 - // This makes us suspect that something else is going wrong. - // Otherwise, we cannot detect a difference in behavior between - // the S3EC V3 Client and the S3 V2 Client with respect to this code. byteRead = stream.read(underlyingBuffer, startPosition, lenToRead); System.out.println("Read " + byteRead + " bytes"); startPosition += byteRead; if (byteRead == 0) { - // Crypto Tools cannot get this case to ever occur. - System.out.println("Looping indefinitely with an encryption client, as startPosition is not increasing"); - break; + // Now we always get this error; we probably were always getting this error, but the log was not writing. + throw new AssertionError( + String.format("Looping indefinitely with an encryption client, as startPosition is not increasing." + + "\n lenToRead: %s \t byteRead: %s \t startPosition: %s", + lenToRead, byteRead, startPosition)); } } }