Skip to content

Commit

Permalink
wip: use localstack
Browse files Browse the repository at this point in the history
  • Loading branch information
@dkubanyi
dkubanyi authored and Daniel Kubanyi committed Jul 19, 2024
1 parent 14ea9a2 commit bf0f78f
Show file tree
Hide file tree
Showing 7 changed files with 96 additions and 128 deletions.
40 changes: 15 additions & 25 deletions gretl/src/integrationTest/java/ch/so/agi/gretl/jobs/S3Bucket2BucketTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,43 +6,33 @@
import ch.so.agi.gretl.util.IntegrationTestUtil;
import org.junit.jupiter.api.Tag;
import org.junit.jupiter.api.Test;
import org.testcontainers.containers.localstack.LocalStackContainer;
import org.testcontainers.junit.jupiter.Container;
import org.testcontainers.junit.jupiter.Testcontainers;
import software.amazon.awssdk.services.s3.S3Client;
import software.amazon.awssdk.services.s3.model.DeleteObjectRequest;
import software.amazon.awssdk.services.s3.model.ListObjectsRequest;
import software.amazon.awssdk.services.s3.model.ListObjectsResponse;
import software.amazon.awssdk.services.s3.model.S3Object;

import java.net.URI;
import java.util.ArrayList;
import java.util.List;

import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.testcontainers.containers.localstack.LocalStackContainer.Service.S3;

@Testcontainers
public class S3Bucket2BucketTest {
@Container
public LocalStackContainer localStackContainer = new LocalStackContainer(S3TestHelper.getLocalstackImage())
.withServices(S3);

class S3Bucket2BucketTest {
private final S3TestHelper s3TestHelper;
private final String s3AccessKey;
private final String s3SecretKey;
private final String s3SourceBucket;
private final String s3TargetBucket;

public S3Bucket2BucketTest() {
this.s3AccessKey = localStackContainer.getAccessKey();
this.s3SecretKey = localStackContainer.getSecretKey();
this.s3AccessKey = System.getProperty("s3AccessKey");
this.s3SecretKey = System.getProperty("s3SecretKey");
this.s3SourceBucket = "ch.so.agi.gretl.test";
this.s3TargetBucket = "ch.so.agi.gretl.test-copy";

URI s3Endpoint = localStackContainer.getEndpointOverride(S3);
String s3Region = localStackContainer.getRegion();
String s3Endpoint = "https://s3.eu-central-1.amazonaws.com";
String s3Region = "eu-central-1";
this.s3TestHelper = new S3TestHelper(this.s3AccessKey, this.s3SecretKey, s3Region, s3Endpoint);
}

Expand All @@ -52,27 +42,27 @@ void uploadDirectory_Ok() throws Exception {
S3Client s3client = s3TestHelper.getS3Client();

s3client.deleteObject(DeleteObjectRequest.builder().bucket(s3TargetBucket).key("foo.txt").build());
s3client.deleteObject(DeleteObjectRequest.builder().bucket(s3TargetBucket).key("bar.txt").build());
s3client.deleteObject(DeleteObjectRequest.builder().bucket(s3TargetBucket).key("bar.txt").build());
s3client.deleteObject(DeleteObjectRequest.builder().bucket(s3TargetBucket).key("download.txt").build());

// Upload files and copy files from one bucket to another.
GradleVariable[] gvs = {
GradleVariable.newGradleProperty("s3AccessKey", s3AccessKey),
GradleVariable[] gvs = {
GradleVariable.newGradleProperty("s3AccessKey", s3AccessKey),
GradleVariable.newGradleProperty("s3SecretKey", s3SecretKey),
GradleVariable.newGradleProperty("s3SourceBucket", s3SourceBucket),
GradleVariable.newGradleProperty("s3TargetBucket", s3TargetBucket)
};
};
IntegrationTestUtil.runJob("src/integrationTest/jobs/S3Bucket2Bucket", gvs);

// Check result.
// Check result.
ListObjectsRequest listObjects = ListObjectsRequest
.builder()
.bucket(s3TargetBucket)
.build();

ListObjectsResponse res = s3client.listObjects(listObjects);
List<S3Object> objects = res.contents();

List<String> keyList = new ArrayList<String>();
for (S3Object myObject : objects) {
keyList.add(myObject.key());
Expand All @@ -82,13 +72,13 @@ void uploadDirectory_Ok() throws Exception {
assertTrue(keyList.contains("bar.txt"));
assertTrue(keyList.contains("download.txt"));
assertEquals(3, keyList.size());

// Remove uploaded files from buckets.
s3client.deleteObject(DeleteObjectRequest.builder().bucket(s3SourceBucket).key("foo.txt").build());
s3client.deleteObject(DeleteObjectRequest.builder().bucket(s3SourceBucket).key("bar.txt").build());

s3client.deleteObject(DeleteObjectRequest.builder().bucket(s3TargetBucket).key("foo.txt").build());
s3client.deleteObject(DeleteObjectRequest.builder().bucket(s3TargetBucket).key("bar.txt").build());
s3client.deleteObject(DeleteObjectRequest.builder().bucket(s3TargetBucket).key("bar.txt").build());
s3client.deleteObject(DeleteObjectRequest.builder().bucket(s3TargetBucket).key("download.txt").build());
}
}
}
}
61 changes: 25 additions & 36 deletions gretl/src/integrationTest/java/ch/so/agi/gretl/jobs/S3DownloadTest.java
View file
Original file line number Diff line number Diff line change
@@ -1,15 +1,15 @@
package ch.so.agi.gretl.jobs;

import ch.so.agi.gretl.testutil.S3TestHelper;
import ch.so.agi.gretl.testutil.TestTags;
import ch.so.agi.gretl.util.GradleVariable;
import ch.so.agi.gretl.util.IntegrationTestUtil;
import org.junit.Test;
import org.junit.jupiter.api.Tag;
import org.junit.jupiter.api.Test;
import org.testcontainers.containers.localstack.LocalStackContainer;
import org.testcontainers.junit.jupiter.Container;
import org.testcontainers.junit.jupiter.Testcontainers;
import software.amazon.awssdk.auth.credentials.AwsBasicCredentials;
import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider;
import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider;
import software.amazon.awssdk.core.ResponseInputStream;
import software.amazon.awssdk.regions.Region;
import software.amazon.awssdk.services.s3.S3Client;
import software.amazon.awssdk.services.s3.model.GetObjectRequest;
import software.amazon.awssdk.services.s3.model.GetObjectResponse;
Expand All @@ -20,55 +20,44 @@
import java.nio.file.Files;
import java.nio.file.Paths;

import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.testcontainers.containers.localstack.LocalStackContainer.Service.S3;
import static org.junit.Assert.assertTrue;

@Testcontainers
public class S3DownloadTest {
@Container
public LocalStackContainer localStackContainer = new LocalStackContainer(S3TestHelper.getLocalstackImage())
.withServices(S3);

private final S3TestHelper s3TestHelper;
private final String s3AccessKey;
private final String s3SecretKey;
private final String s3BucketName;

public S3DownloadTest() {
this.s3AccessKey = localStackContainer.getAccessKey();
this.s3SecretKey = localStackContainer.getSecretKey();
this.s3BucketName = System.getProperty("s3BucketName");

final URI s3Endpoint = localStackContainer.getEndpointOverride(S3);
final String s3Region = localStackContainer.getRegion();
this.s3TestHelper = new S3TestHelper(this.s3AccessKey, this.s3SecretKey, s3Region, s3Endpoint);
}

private final String s3AccessKey = System.getProperty("s3AccessKey");
private final String s3SecretKey = System.getProperty("s3SecretKey");
private final String s3BucketName = System.getProperty("s3BucketName");

@Test
@Tag(TestTags.S3_TEST)
void downloadFile_Ok() throws Exception {
public void downloadFile_Ok() throws Exception {
// Download single file from a directory.
GradleVariable[] gvs = {
GradleVariable.newGradleProperty("s3AccessKey", s3AccessKey),
GradleVariable[] gvs = {
GradleVariable.newGradleProperty("s3AccessKey", s3AccessKey),
GradleVariable.newGradleProperty("s3SecretKey", s3SecretKey),
GradleVariable.newGradleProperty("s3BucketName", s3BucketName)
};
};
IntegrationTestUtil.runJob("src/integrationTest/jobs/S3DownloadFile", gvs);

// Check result.
S3Client s3client = s3TestHelper.getS3Client();
AwsCredentialsProvider creds = StaticCredentialsProvider.create(AwsBasicCredentials.create(s3AccessKey, s3SecretKey));
Region region = Region.of("eu-central-1");
S3Client s3client = S3Client.builder()
.credentialsProvider(creds)
.region(region)
.endpointOverride(URI.create("https://s3.eu-central-1.amazonaws.com"))
.build();

GetObjectRequest getObjectRequest = GetObjectRequest.builder()
.bucket(s3BucketName)
.key("download.txt")
.build();

ResponseInputStream<GetObjectResponse> is = s3client.getObject(getObjectRequest);
BufferedReader reader = new BufferedReader(new InputStreamReader(is));
BufferedReader reader = new BufferedReader(new InputStreamReader(is));
assertTrue(reader.readLine().equalsIgnoreCase("fubar"));

// Remove downloaded file.
Files.delete(Paths.get("src/integrationTest/jobs/S3DownloadFile/download.txt"));
}
}
}
}
Loading

0 comments on commit bf0f78f

Please sign in to comment.