Skip to content

Commit

Permalink
feat: use localstack for AWS testcontainers
Browse files Browse the repository at this point in the history
  • Loading branch information
@dkubanyi
dkubanyi authored and Daniel Kubanyi committed Jul 18, 2024
1 parent 4120b08 commit cbe47a7
Show file tree
Hide file tree
Showing 6 changed files with 97 additions and 54 deletions.
2 changes: 2 additions & 0 deletions gradle/libs.versions.toml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,7 @@ gradleSshPluginVersion = "2.9.0"
gradleAwsPluginVersion = "0.41"
pluginPublishVersion = "0.21.0"
cyclonedxBomVersion = "1.7.4"
awssdk = "1.12.761"

[libraries]
iox-ili = { group = "ch.interlis", name = "iox-ili", version.ref = "ioxIliVersion" }
Expand All @@ -56,6 +57,7 @@ iox-formats = { group = "io.github.sogis", name = "iox-formats", version.ref = "
av2-ch = { group = "io.github.sogis", name = "av2ch", version.ref = "av2chVersion" }
av2-geobau = { group = "ch.interlis", name = "av2geobau", version.ref = "av2geobauVersion" }
aws-sdk-s3 = { group = "software.amazon.awssdk", name = "s3", version.ref = "awsSdkVersion" }
aws-java-sdk = { group = "com.amazonaws", name = "aws-java-sdk", version.ref = "awssdk" }
apache-commons-io = { group = "commons-io", name = "commons-io", version.ref = "apacheCommonsIoVersion" }
apache-commons-net = { group = "commons-net", name = "commons-net", version.ref = "apacheCommonsNetVersion" }
apache-commons-code = { group = "commons-codec", name = "commons-codec", version.ref = "apacheCommonsCodeVersion" }
Expand Down
1 change: 1 addition & 0 deletions gretl/build.gradle
View file
Original file line number Diff line number Diff line change
Expand Up @@ -125,6 +125,7 @@ dependencies {
testImplementation libs.test.containers.base
testImplementation libs.test.containers.postgresql
testImplementation libs.test.containers.localstack
testImplementation libs.aws.java.sdk

testImplementation libs.mock.web.server

Expand Down
52 changes: 27 additions & 25 deletions gretl/src/test/java/ch/so/agi/gretl/steps/S3Bucket2BucketStepTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
import ch.so.agi.gretl.testutil.S3TestHelper;
import ch.so.agi.gretl.testutil.TestTags;
import ch.so.agi.gretl.testutil.TestUtil;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Tag;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.io.TempDir;
Expand All @@ -29,40 +30,43 @@
@Testcontainers
public class S3Bucket2BucketStepTest {
@Container
public LocalStackContainer localStackContainer = new LocalStackContainer(S3TestHelper.getLocalstackImage())
public static LocalStackContainer localStackContainer = new LocalStackContainer(S3TestHelper.getLocalstackImage())
.withServices(S3);

private final String s3AccessKey;
private final String s3SecretKey;
private final String s3SourceBucketName;
private final String s3TargetBucketName;
private final URI s3Endpoint;
private final String s3Region;
private final String acl;
private final S3TestHelper s3TestHelper;
private static String s3AccessKey;
private static String s3SecretKey;
private static String s3SourceBucketName;
private static String s3TargetBucketName;
private static URI s3Endpoint;
private static String s3Region;
private static String acl;
private static S3TestHelper s3TestHelper;

@TempDir
public Path folder;

public S3Bucket2BucketStepTest() {
this.s3AccessKey = localStackContainer.getAccessKey();
this.s3SecretKey = localStackContainer.getSecretKey();
this.s3SourceBucketName = "ch.so.agi.gretl.test";
this.s3TargetBucketName = "ch.so.agi.gretl.test-copy";
this.s3Endpoint = localStackContainer.getEndpointOverride(S3);
this.s3Region = localStackContainer.getRegion();
this.acl = "public-read";
this.s3TestHelper = new S3TestHelper(this.s3AccessKey, this.s3SecretKey, this.s3Region, this.s3Endpoint);
@BeforeAll
public static void setUpClass() {
s3AccessKey = localStackContainer.getAccessKey();
s3SecretKey = localStackContainer.getSecretKey();
s3SourceBucketName = "ch.so.agi.gretl.test";
s3TargetBucketName = "ch.so.agi.gretl.test-copy";
s3Endpoint = localStackContainer.getEndpointOverride(S3);
s3Region = localStackContainer.getRegion();
acl = "public-read";
s3TestHelper = new S3TestHelper(s3AccessKey, s3SecretKey, s3Region, s3Endpoint);
}

@Test
@Tag(TestTags.S3_TEST)
public void copyFiles_Ok() throws Exception {
File sourceObject = TestUtil.getResourceFile(TestUtil.S3_BUCKET_DIR_PATH);
Map<String,String> metadata = new HashMap<>();
S3Client s3Client = s3TestHelper.getS3Client();
s3TestHelper.createBucketIfNotExists(s3Client, s3SourceBucketName);
s3TestHelper.createBucketIfNotExists(s3Client, s3TargetBucketName);

deleteObjects(s3Client, Arrays.asList("foo.txt", "bar.txt", "download.txt"));
File sourceObject = TestUtil.getResourceFile(TestUtil.S3_BUCKET_DIR_PATH);
Map<String,String> metadata = new HashMap<>();
deleteObjects(s3Client, Arrays.asList("foo.txt", "bar.txt"));
s3TestHelper.upload(sourceObject, metadata, s3SourceBucketName, acl);
copyFiles(metadata);

Expand All @@ -78,8 +82,7 @@ public void copyFiles_Ok() throws Exception {

assertTrue(keyList.contains("foo.txt"));
assertTrue(keyList.contains("bar.txt"));
assertTrue(keyList.contains("download.txt"));
assertEquals(3, keyList.size());
assertEquals(2, keyList.size());
deleteFiles(s3Client);
}

Expand Down Expand Up @@ -121,6 +124,5 @@ private void deleteFiles(S3Client s3Client) {
s3Client.deleteObject(DeleteObjectRequest.builder().bucket(s3SourceBucketName).key("bar.txt").build());
s3Client.deleteObject(DeleteObjectRequest.builder().bucket(s3TargetBucketName).key("foo.txt").build());
s3Client.deleteObject(DeleteObjectRequest.builder().bucket(s3TargetBucketName).key("bar.txt").build());
s3Client.deleteObject(DeleteObjectRequest.builder().bucket(s3TargetBucketName).key("download.txt").build());
}
}
44 changes: 32 additions & 12 deletions gretl/src/test/java/ch/so/agi/gretl/steps/S3DownloadStepTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,57 +5,77 @@
import ch.so.agi.gretl.testutil.S3TestHelper;
import ch.so.agi.gretl.testutil.TestTags;
import ch.so.agi.gretl.testutil.TestUtil;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Tag;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.io.TempDir;
import org.testcontainers.containers.localstack.LocalStackContainer;
import org.testcontainers.junit.jupiter.Container;
import org.testcontainers.junit.jupiter.Testcontainers;
import software.amazon.awssdk.services.s3.S3Client;

import java.io.File;
import java.net.URI;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.HashMap;

import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.testcontainers.containers.localstack.LocalStackContainer.Service.S3;

@Testcontainers
public class S3DownloadStepTest {
@Container
public LocalStackContainer localStackContainer = new LocalStackContainer(S3TestHelper.getLocalstackImage())
public static LocalStackContainer localStackContainer = new LocalStackContainer(S3TestHelper.getLocalstackImage())
.withServices(S3);

private final String s3AccessKey;
private final String s3SecretKey;
private final String s3BucketName;
private static String s3AccessKey;
private static String s3SecretKey;
private static String s3BucketName;
private static URI s3Endpoint;
private static String s3Region;
private static S3TestHelper s3TestHelper;

private final GretlLogger log;

@TempDir
public Path folder;

public S3DownloadStepTest() {
this.log = LogEnvironment.getLogger(this.getClass());
this.s3AccessKey = localStackContainer.getAccessKey();
this.s3SecretKey = localStackContainer.getSecretKey();
this.s3BucketName = System.getProperty("s3BucketName");
}

@BeforeAll
public static void setUpClass() throws Exception {
s3AccessKey = localStackContainer.getAccessKey();
s3SecretKey = localStackContainer.getSecretKey();
s3BucketName = System.getProperty("s3BucketName");
s3Endpoint = localStackContainer.getEndpointOverride(S3);
s3Region = localStackContainer.getRegion();
s3TestHelper = new S3TestHelper(s3AccessKey, s3SecretKey, s3Region, s3Endpoint);
}

@Test
@Tag(TestTags.S3_TEST)
public void downloadFile_Ok() throws Exception {
URI s3EndPoint = localStackContainer.getEndpointOverride(S3);
String s3Region = localStackContainer.getRegion();
String key = "download.txt";
S3Client s3Client = s3TestHelper.getS3Client();
s3TestHelper.createBucketIfNotExists(s3Client, s3BucketName);

String key = "foo.txt";
Path downloadDir = TestUtil.createTempDir(folder, "downloadFile_Ok");

// Upload the file
File sourceObject = TestUtil.getResourceFile("data/s3upload/foo.txt");
s3TestHelper.upload(sourceObject, new HashMap<>(), s3BucketName, "public-read");

// Download a single file.
S3DownloadStep s3DownloadStep = new S3DownloadStep();
s3DownloadStep.execute(s3AccessKey, s3SecretKey, s3BucketName, key, s3EndPoint.toString(), s3Region, downloadDir.toFile());
s3DownloadStep.execute(s3AccessKey, s3SecretKey, s3BucketName, key, s3Endpoint.toString(), s3Region, downloadDir.toFile());

// Check result.
String content = new String(Files.readAllBytes(Paths.get(downloadDir.toAbsolutePath().toString(), key)));
log.debug(content);
assertEquals("fubar", content.substring(0, 5));
assertEquals("foo", content.substring(0, 3));
}
}
41 changes: 24 additions & 17 deletions gretl/src/test/java/ch/so/agi/gretl/steps/S3UploadStepTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
import ch.so.agi.gretl.testutil.S3TestHelper;
import ch.so.agi.gretl.testutil.TestTags;
import ch.so.agi.gretl.testutil.TestUtil;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Tag;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.io.TempDir;
Expand All @@ -29,30 +30,31 @@
@Testcontainers
public class S3UploadStepTest {
@Container
public LocalStackContainer localStackContainer = new LocalStackContainer(S3TestHelper.getLocalstackImage())
public static LocalStackContainer localStackContainer = new LocalStackContainer(S3TestHelper.getLocalstackImage())
.withServices(S3);

private final String s3AccessKey;
private final String s3SecretKey;
private final String s3BucketName;
private final URI s3Endpoint;
private final String s3Region;
private final String acl;
private final S3TestHelper s3TestHelper;
private static String s3AccessKey;
private static String s3SecretKey;
private static String s3BucketName;
private static URI s3Endpoint;
private static String s3Region;
private static String acl;
private static S3TestHelper s3TestHelper;

@TempDir
public Path folder;

public S3UploadStepTest() {
this.s3AccessKey = localStackContainer.getAccessKey();
this.s3SecretKey = localStackContainer.getSecretKey();
this.s3BucketName = System.getProperty("s3BucketName");
this.s3Endpoint = localStackContainer.getEndpointOverride(S3);
this.s3Region = localStackContainer.getRegion();
this.acl = "public-read";
this.s3TestHelper = new S3TestHelper(this.s3AccessKey, this.s3SecretKey, this.s3Region, this.s3Endpoint);
@BeforeAll
public static void setUp() throws Exception {
s3AccessKey = localStackContainer.getAccessKey();
s3SecretKey = localStackContainer.getSecretKey();
s3BucketName = "emptybucket"; // System.getProperty("s3BucketName");
s3Endpoint = localStackContainer.getEndpointOverride(S3);
s3Region = localStackContainer.getRegion();
acl = "public-read";
s3TestHelper = new S3TestHelper(s3AccessKey, s3SecretKey, s3Region, s3Endpoint);
}

@Test
@Tag(TestTags.S3_TEST)
public void uploadDirectory_Ok() throws Exception {
Expand All @@ -62,6 +64,8 @@ public void uploadDirectory_Ok() throws Exception {
}};

S3Client s3Client = s3TestHelper.getS3Client();
s3TestHelper.createBucketIfNotExists(s3Client, s3BucketName);

s3Client.deleteObject(DeleteObjectRequest.builder().bucket(s3BucketName).key("foo.txt").build());
s3Client.deleteObject(DeleteObjectRequest.builder().bucket(s3BucketName).key("bar.txt").build());

Expand Down Expand Up @@ -97,6 +101,7 @@ public void uploadFile_Ok() throws Exception {
File sourceObject = TestUtil.getResourceFile("data/s3upload/foo.txt");
Map<String,String> metaData = new HashMap<>();
S3Client s3Client = s3TestHelper.getS3Client();
s3TestHelper.createBucketIfNotExists(s3Client, s3BucketName);

s3Client.deleteObject(DeleteObjectRequest.builder().bucket(s3BucketName).key("foo.txt").build());

Expand All @@ -122,6 +127,8 @@ public void uploadFile_Ok() throws Exception {
public void uploadFile_Fail() throws Exception {
File sourceObject = TestUtil.getResourceFile("data/s3upload/foo.txt");
Map<String,String> metaData = new HashMap<>();
S3Client s3Client = s3TestHelper.getS3Client();
s3TestHelper.createBucketIfNotExists(s3Client, s3BucketName);

try {
// Upload a single file
Expand Down
11 changes: 11 additions & 0 deletions gretl/src/test/java/ch/so/agi/gretl/testutil/S3TestHelper.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,9 @@
import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider;
import software.amazon.awssdk.regions.Region;
import software.amazon.awssdk.services.s3.S3Client;
import software.amazon.awssdk.services.s3.model.CreateBucketRequest;
import software.amazon.awssdk.services.s3.model.HeadBucketRequest;
import software.amazon.awssdk.services.s3.model.NoSuchBucketException;

import java.io.File;
import java.io.FileNotFoundException;
Expand Down Expand Up @@ -60,6 +63,14 @@ public S3Client getS3Client() {
.build();
}

public void createBucketIfNotExists(S3Client client, String bucketName) {
try {
client.headBucket(HeadBucketRequest.builder().bucket(bucketName).build());
} catch (NoSuchBucketException e) {
client.createBucket(CreateBucketRequest.builder().bucket(bucketName).build());
}
}

private AwsCredentialsProvider getCredentialsProvider() {
return StaticCredentialsProvider.create(
AwsBasicCredentials.create(s3AccessKey, s3SecretKey)
Expand Down

0 comments on commit cbe47a7

Please sign in to comment.