Tony/refactor tests

.github/workflows/ghi_300.yml

@@ -0,0 +1,49 @@
+name: ghi_300.yml
+on:
+  push:
+    branches:
+      - tony/refactor-tests
+      - 'ghi-300/**'
+
+jobs:
+  Build:
+    runs-on: ubuntu-latest
+    permissions:
+      id-token: write
+      contents: read
+
+    steps:
+      - name: Configure AWS Credentials
+        uses: aws-actions/configure-aws-credentials@v2
+        with:
+          role-to-assume: arn:aws:iam::${{ secrets.CI_AWS_ACCOUNT_ID }}:role/service-role/${{ vars.CI_AWS_ROLE }}
+          role-session-name: S3EC-Github-CI-Tests
+          aws-region: ${{ vars.CI_AWS_REGION }}
+
+      - name: Checkout Code
+        uses: actions/checkout@v3
+
+      # TODO: Add OpenJDK
+      # OpenJDK would require a different action than setup-java, so setup is more involved.
+
+      - name: Setup JDK
+        uses: actions/setup-java@v3
+        with:
+          distribution: corretto
+          java-version: 8
+          cache: 'maven'
+
+      - name: Compile
+        run: |
+          mvn --batch-mode -no-transfer-progress clean compile
+          mvn --batch-mode -no-transfer-progress test-compile
+        shell: bash
+
+      - name: Test
+        run: |
+          export AWS_S3EC_TEST_BUCKET=${{ vars.CI_S3_BUCKET }}
+          export AWS_S3EC_TEST_KMS_KEY_ID=arn:aws:kms:${{ vars.CI_AWS_REGION }}:${{ secrets.CI_AWS_ACCOUNT_ID }}:key/${{ vars.CI_KMS_KEY_ID }}
+          export AWS_S3EC_TEST_KMS_KEY_ALIAS=arn:aws:kms:${{ vars.CI_AWS_REGION }}:${{ secrets.CI_AWS_ACCOUNT_ID }}:alias/${{ vars.CI_KMS_KEY_ALIAS }}
+          export AWS_REGION=${{ vars.CI_AWS_REGION }}
+          mvn -B -ntp -DskipCompile -Dtest=software.amazon.encryption.s3.examples.TestEndOfStreamBehavior test
+        shell: bash

pom.xml

@@ -41,6 +41,7 @@
     <maven.compiler.target>8</maven.compiler.target>
     <maven.compiler.release>8</maven.compiler.release>
     <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+    <aws.java.sdk.version>2.26.7</aws.java.sdk.version>
   </properties>
 
   <dependencyManagement>
@@ -56,7 +57,7 @@
       <dependency>
         <groupId>software.amazon.awssdk</groupId>
         <artifactId>bom</artifactId>
-        <version>2.20.38</version>
+        <version>${aws.java.sdk.version}</version>
         <optional>true</optional>
         <type>pom</type>
         <scope>import</scope>
@@ -68,21 +69,20 @@
     <dependency>
       <groupId>software.amazon.awssdk</groupId>
       <artifactId>s3</artifactId>
-      <version>2.20.38</version>
+      <version>${aws.java.sdk.version}</version>
     </dependency>
 
     <dependency>
       <groupId>software.amazon.awssdk</groupId>
       <artifactId>kms</artifactId>
-      <version>2.20.38</version>
+      <version>${aws.java.sdk.version}</version>
       <optional>true</optional>
     </dependency>
 
-    <!-- Used when enableMultipartPutObject is configured -->
     <dependency>
-      <groupId>software.amazon.awssdk.crt</groupId>
-      <artifactId>aws-crt</artifactId>
-      <version>0.29.24</version>
+      <groupId>software.amazon.awssdk</groupId>
+      <artifactId>aws-crt-client</artifactId>
+      <version>${aws.java.sdk.version}</version>
       <optional>true</optional>
     </dependency>
 

src/test/java/software/amazon/encryption/s3/S3EncryptionClientCompatibilityTest.java

@@ -28,6 +28,7 @@
 import software.amazon.awssdk.services.s3.model.GetObjectRequest;
 import software.amazon.awssdk.services.s3.model.GetObjectResponse;
 import software.amazon.awssdk.services.s3.model.PutObjectRequest;
+import software.amazon.encryption.s3.utils.S3EncryptionClientTestResources;
 
 import javax.crypto.KeyGenerator;
 import javax.crypto.SecretKey;
@@ -37,6 +38,7 @@
 import java.security.KeyPair;
 import java.security.KeyPairGenerator;
 import java.security.NoSuchAlgorithmException;
+import java.security.spec.InvalidKeySpecException;
 import java.util.HashMap;
 import java.util.Map;
 
@@ -45,29 +47,28 @@
 import static software.amazon.encryption.s3.S3EncryptionClient.withAdditionalConfiguration;
 import static software.amazon.encryption.s3.utils.S3EncryptionClientTestResources.appendTestSuffix;
 import static software.amazon.encryption.s3.utils.S3EncryptionClientTestResources.deleteObject;
+import static software.amazon.encryption.s3.utils.S3EncryptionClientTestResources.BUCKET;
+import static software.amazon.encryption.s3.utils.S3EncryptionClientTestResources.KMS_KEY_ID;
 
 /**
  * This class is an integration test for verifying compatibility of ciphertexts
  * between V1, V2, and V3 clients under various conditions.
  */
 public class S3EncryptionClientCompatibilityTest {
-
-    private static final String BUCKET = System.getenv("AWS_S3EC_TEST_BUCKET");
-    private static final String KMS_KEY_ID = System.getenv("AWS_S3EC_TEST_KMS_KEY_ID");
-    private static final Region KMS_REGION = Region.getRegion(Regions.fromName(System.getenv("AWS_REGION")));
+    // SDK V1 Region
+    private static final Region KMS_REGION = Region.getRegion(
+        Regions.fromName(S3EncryptionClientTestResources.KMS_REGION.toString()));
 
     private static SecretKey AES_KEY;
     private static KeyPair RSA_KEY_PAIR;
 
     @BeforeAll
-    public static void setUp() throws NoSuchAlgorithmException {
+    public static void setUp() throws NoSuchAlgorithmException, IOException, InvalidKeySpecException {
         KeyGenerator keyGen = KeyGenerator.getInstance("AES");
         keyGen.init(256);
         AES_KEY = keyGen.generateKey();
 
-        KeyPairGenerator keyPairGen = KeyPairGenerator.getInstance("RSA");
-        keyPairGen.initialize(2048);
-        RSA_KEY_PAIR = keyPairGen.generateKeyPair();
+        RSA_KEY_PAIR = S3EncryptionClientTestResources.getRSAKeyPair();
     }
 
     @Test

src/test/java/software/amazon/encryption/s3/examples/TestEndOfStreamBehavior.java

@@ -0,0 +1,127 @@
+package software.amazon.encryption.s3.examples;
+
+import java.io.InputStream;
+import java.nio.ByteBuffer;
+import java.security.KeyPair;
+import java.util.stream.Stream;
+
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.MethodSource;
+import software.amazon.awssdk.core.sync.RequestBody;
+import software.amazon.awssdk.http.async.SdkAsyncHttpClient;
+import software.amazon.awssdk.http.nio.netty.NettyNioAsyncHttpClient;
+import software.amazon.awssdk.regions.Region;
+import software.amazon.awssdk.services.s3.S3AsyncClient;
+import software.amazon.awssdk.services.s3.S3Client;
+import software.amazon.awssdk.services.s3.model.DeleteObjectRequest;
+import software.amazon.awssdk.services.s3.model.GetObjectRequest;
+import software.amazon.awssdk.services.s3.model.PutObjectRequest;
+import software.amazon.encryption.s3.S3EncryptionClient;
+import software.amazon.encryption.s3.utils.S3EncryptionClientTestResources;
+
+import static software.amazon.encryption.s3.utils.S3EncryptionClientTestResources.*;
+
+public class TestEndOfStreamBehavior {
+    private static final Region DEFAULT_REGION = KMS_REGION;
+    private static final String KEY = "GHI-300.txt";
+    @SuppressWarnings("SpellCheckingInspection")
+    private static final byte[] CONTENT = new String(new char[4])
+        .replace("\0", "abcdefghijklmnopqrstuvwxyz0123456789")
+        .getBytes();
+    /** The encryption key to use in client-side encryption tests. */
+    protected static final KeyPair KEY_PAIR;
+
+    static {
+        try {
+            KEY_PAIR = S3EncryptionClientTestResources.getRSAKeyPair();
+        } catch (Exception e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    static Stream<S3Client> clientProvider() {
+        return Stream.of(
+            getClient(DEFAULT_REGION),
+            getEncryptionClient(KEY_PAIR, DEFAULT_REGION));
+    }
+
+    @ParameterizedTest
+    @MethodSource("clientProvider")
+    void testEndOfStreamBehavior(final S3Client client) throws Exception {
+        // Delete the data if it exists
+        final DeleteObjectRequest deleteRequest = DeleteObjectRequest.builder()
+            .bucket(BUCKET)
+            .key(KEY)
+            .build();
+
+        client.deleteObject(deleteRequest);
+
+        // Upload the data
+        final PutObjectRequest uploadRequest =
+            PutObjectRequest.builder().bucket(BUCKET).key(KEY).build();
+        client.putObject(uploadRequest, RequestBody.fromBytes(CONTENT));
+        // wait 5 seconds for the data to be uploaded
+        Thread.sleep(5000);
+
+        // Actual test
+        final GetObjectRequest downloadRequest =
+            GetObjectRequest.builder()
+                .bucket(BUCKET)
+                .key(KEY)
+                .range("bytes=0-15")
+                .build();
+
+        final InputStream stream = client.getObject(downloadRequest);
+
+        // Buffer capacity matters !!!
+        // Behavior difference when the capacity is same as the content length (i.e. 16) of the ranged query
+        final ByteBuffer buffer = ByteBuffer.allocate(16);
+        final byte[] underlyingBuffer = buffer.array();
+        final int capacity = buffer.capacity();
+
+        final int END_OF_STREAM = -1;
+        int byteRead = 0;
+        int startPosition = 0;
+        while (byteRead != END_OF_STREAM) {
+            int lenToRead = capacity - startPosition;
+            System.out.println("Start position: " + startPosition + " Length to read: " + lenToRead);
+            byteRead = stream.read(underlyingBuffer, startPosition, lenToRead);
+            System.out.println("Read " + byteRead + " bytes");
+            startPosition += byteRead;
+            if (byteRead == 0) {
+                // Now we always get this error; we probably were always getting this error, but the log was not writing.
+                throw new AssertionError(
+                    String.format("Looping indefinitely with an encryption client, as startPosition is not increasing." +
+                            "\n lenToRead: %s \t byteRead: %s \t startPosition: %s",
+                        lenToRead, byteRead, startPosition));
+            }
+        }
+    }
+
+    public static S3Client getEncryptionClient(final KeyPair keyPair, final Region region) {
+        return S3EncryptionClient.builder()
+            .rsaKeyPair(keyPair)
+            .enableLegacyUnauthenticatedModes(true)
+            .wrappedClient(getClient(region))
+            .wrappedAsyncClient(getAsyncClient(region))
+            .build();
+    }
+
+    public static S3Client getClient(final Region region) {
+        return S3Client.builder()
+            .region(region)
+            .credentialsProvider(CREDENTIALS)
+            .httpClient(HTTP_CLIENT)
+            .build();
+    }
+
+    public static S3AsyncClient getAsyncClient(final Region region) {
+        final SdkAsyncHttpClient nettyHttpClient =
+            NettyNioAsyncHttpClient.builder().maxConcurrency(100).build();
+        return S3AsyncClient.builder()
+            .region(region)
+            .credentialsProvider(CREDENTIALS)
+            .httpClient(nettyHttpClient)
+            .build();
+    }
+}

src/test/java/software/amazon/encryption/s3/utils/S3EncryptionClientTestResources.java

@@ -2,23 +2,44 @@
 // SPDX-License-Identifier: Apache-2.0
 package software.amazon.encryption.s3.utils;
 
+import org.bouncycastle.util.io.pem.PemObject;
+import org.bouncycastle.util.io.pem.PemReader;
+import org.bouncycastle.util.io.pem.PemWriter;
 import org.joda.time.DateTime;
 import org.joda.time.format.DateTimeFormat;
+import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider;
+import software.amazon.awssdk.auth.credentials.DefaultCredentialsProvider;
+import software.amazon.awssdk.http.SdkHttpClient;
+import software.amazon.awssdk.http.apache.ApacheHttpClient;
+import software.amazon.awssdk.regions.Region;
 import software.amazon.awssdk.services.s3.S3AsyncClient;
 import software.amazon.awssdk.services.s3.S3Client;
 import software.amazon.awssdk.services.s3.model.DeleteObjectResponse;
 
+import java.io.*;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.security.*;
+import java.security.interfaces.RSAPrivateKey;
+import java.security.interfaces.RSAPublicKey;
+import java.security.spec.InvalidKeySpecException;
+import java.security.spec.PKCS8EncodedKeySpec;
+import java.security.spec.X509EncodedKeySpec;
 import java.util.concurrent.CompletableFuture;
 
 /**
  * Determines which AWS resources to use while running tests.
  */
 public class S3EncryptionClientTestResources {
 
+    public static final AwsCredentialsProvider CREDENTIALS = DefaultCredentialsProvider.create();
+    public static final SdkHttpClient HTTP_CLIENT = ApacheHttpClient.create();
     public static final String BUCKET = System.getenv("AWS_S3EC_TEST_BUCKET");
     public static final String KMS_KEY_ID = System.getenv("AWS_S3EC_TEST_KMS_KEY_ID");
     // This alias must point to the same key as KMS_KEY_ID
     public static final String KMS_KEY_ALIAS = System.getenv("AWS_S3EC_TEST_KMS_KEY_ALIAS");
+    public static final Region KMS_REGION = Region.of(System.getenv("AWS_REGION"));
 
     /**
      * For a given string, append a suffix to distinguish it from
@@ -57,4 +78,63 @@ public static void deleteObject(final String bucket, final String objectKey, fin
         // Ensure completion before return
         response.join();
     }
+
+
+    /**
+     * @return If an RSA KeyPair already exists in Test Resources, load and return that.<p>
+     * Otherwise, generate a new key pair, persist that to Resources, and return it.<p>
+     * Assumes working directory is root of the git repo.
+     */
+    public static KeyPair getRSAKeyPair() throws IOException, NoSuchAlgorithmException, InvalidKeySpecException {
+        Path resourceDirectory = Paths.get("src","test","resources");
+        if (resourceDirectory.resolve("RSAPrivateKey.pem").toFile().exists() && resourceDirectory.resolve("RSAPublicKey.pem").toFile().exists()) {
+            return readKeyPairFromTestResourcesFile(resourceDirectory);
+        }
+        KeyPair keyPair = generateKeyPair(2048);
+        writeKeyPairToTestResourcesFile(keyPair, resourceDirectory);
+        return keyPair;
+    }
+
+    public static KeyPair generateKeyPair(final int keySize) {
+        if (!(keySize == 2048 || keySize == 4096)) throw new IllegalArgumentException("Only 2048 or 4096 are valid key sizes.");
+        KeyPairGenerator rsaGen;
+        try {
+            rsaGen = KeyPairGenerator.getInstance("RSA");
+        } catch (NoSuchAlgorithmException e) {
+            throw new RuntimeException("No such algorithm", e);
+        }
+        rsaGen.initialize(keySize, new SecureRandom());
+        return rsaGen.generateKeyPair();
+    }
+
+    private static void writePEMFile(Key key, String description, Path filePath) throws IOException {
+        final PemObject pemObject = new PemObject(description, key.getEncoded());
+        try (PemWriter pemWriter = new PemWriter(new OutputStreamWriter(Files.newOutputStream(filePath)))) {
+            pemWriter.writeObject(pemObject);
+        }
+    }
+
+    private static PemObject readPEMFile(Path filePath) throws IOException {
+        try (PemReader pemReader = new PemReader(new InputStreamReader(Files.newInputStream(filePath)))) {
+            return pemReader.readPemObject();
+        }
+    }
+
+    private static void writeKeyPairToTestResourcesFile(final KeyPair keyPair, Path resourceDirectory) throws IOException {
+        RSAPrivateKey privateKey = (RSAPrivateKey) keyPair.getPrivate();
+        writePEMFile(privateKey, "RSA PRIVATE KEY", resourceDirectory.resolve("RSAPrivateKey.pem"));
+        RSAPublicKey publicKey = (RSAPublicKey) keyPair.getPublic();
+        writePEMFile(publicKey, "RSA PUBLIC KEY", resourceDirectory.resolve("RSAPublicKey.pem"));
+    }
+
+    private static KeyPair readKeyPairFromTestResourcesFile(Path resourceDirectory) throws IOException, NoSuchAlgorithmException, InvalidKeySpecException {
+        final KeyFactory factory = KeyFactory.getInstance("RSA");
+        byte[] privateKeyContent = readPEMFile(resourceDirectory.resolve("RSAPrivateKey.pem")).getContent();
+        PKCS8EncodedKeySpec privateKeySpec = new PKCS8EncodedKeySpec(privateKeyContent);
+        final PrivateKey privateKey = factory.generatePrivate(privateKeySpec);
+        byte[] publicKeyContent = readPEMFile(resourceDirectory.resolve("RSAPublicKey.pem")).getContent();
+        X509EncodedKeySpec publicKeySpec = new X509EncodedKeySpec(publicKeyContent);
+        final PublicKey publicKey = factory.generatePublic(publicKeySpec);
+        return new KeyPair(publicKey, privateKey);
+    }
 }