Skip to content

Commit

Permalink
feat: homogenize test code
Browse files Browse the repository at this point in the history 
Signed-off-by: Daniel Kubányi <[email protected]>
  • Loading branch information
@dkubanyi
dkubanyi committed Jun 13, 2024
1 parent 2815a66 commit cc2fe0f
Show file tree
Hide file tree
Showing 9 changed files with 188 additions and 165 deletions.
13 changes: 6 additions & 7 deletions gretl/src/test/java/ch/so/agi/gretl/steps/Db2DbStepTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -242,13 +242,12 @@ public void closeConnectionsAfterFailedTest_connectionsClosed() throws Exception
Db2DbStep db2db = new Db2DbStep();

Assert.assertThrows(SQLException.class, () -> db2db.processAllTransferSets(sourceDb, targetDb, transferSets));
boolean isclosed = sourceDb.isClosed();
Assert.assertTrue("SourceConnection is not closed", sourceDb.isClosed());
Assert.assertTrue("TargetConnection is not closed", targetDb.isClosed());
}

@Category(DbTest.class)
@Test
@Category(DbTest.class)
public void canWriteGeomFromWkbTest_Ok() throws Exception {
String schemaName = "GeomFromWkbTest";

Expand Down Expand Up @@ -278,8 +277,8 @@ public void canWriteGeomFromWkbTest_Ok() throws Exception {
}
}

@Category(DbTest.class)
@Test
@Category(DbTest.class)
public void canWriteGeomFromWktTest_Ok() throws Exception {
String schemaName = "GeomFromWktTest";

Expand Down Expand Up @@ -309,8 +308,8 @@ public void canWriteGeomFromWktTest_Ok() throws Exception {
}
}

@Category(DbTest.class)
@Test
@Category(DbTest.class)
public void canWriteGeomFromGeoJsonTest_Ok() throws Exception {
String schemaName = "GeomFromGeoJsonTest";

Expand Down Expand Up @@ -343,8 +342,8 @@ public void canWriteGeomFromGeoJsonTest_Ok() throws Exception {
* Test's loading several hundred thousand rows from sqlite to postgis. Loading
* 300'000 rows should take about 15 seconds
*/
@Category(DbTest.class)
@Test
@Category(DbTest.class)
public void positiveBulkLoadPostgisTest_Ok() throws Exception {
int numRows = 300000;
String schemaName = "BULKLOAD2POSTGIS";
Expand Down Expand Up @@ -390,8 +389,8 @@ public void positiveBulkLoadPostgisTest_Ok() throws Exception {
* Tests if the sqlite datatypes and geometry as wkt are transferred faultfree
* from sqlite to postgis
*/
@Category(DbTest.class)
@Test
@Category(DbTest.class)
public void positiveSqlite2PostgisTest_Ok() throws Exception {
String schemaName = "SQLITE2POSTGIS";
File sqliteDb = createTmpDb(schemaName);
Expand Down Expand Up @@ -438,8 +437,8 @@ public void positiveSqlite2PostgisTest_Ok() throws Exception {
* Tests if the "special" datatypes (Date, Time, GUID, Geometry, ..) are
* transferred faultfree from Postgis to sqlite
*/
@Category(DbTest.class)
@Test
@Category(DbTest.class)
public void positivePostgis2SqliteTest_Ok() throws Exception {
String schemaName = "POSTGIS2SQLITE";
File sqliteDb = createTmpDb(schemaName);
Expand Down
18 changes: 7 additions & 11 deletions gretl/src/test/java/ch/so/agi/gretl/steps/PostgisRasterExportStepTest.java
View file
Original file line number Diff line number Diff line change
@@ -1,21 +1,17 @@
package ch.so.agi.gretl.steps;

import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;

import java.io.File;
import java.nio.file.Paths;

import ch.so.agi.gretl.api.Connector;
import ch.so.agi.gretl.testutil.TestUtil;
import org.junit.*;
import org.junit.rules.TemporaryFolder;
import org.testcontainers.containers.PostgisContainerProvider;
import org.testcontainers.containers.PostgreSQLContainer;
import org.testcontainers.containers.wait.strategy.Wait;

import ch.so.agi.gretl.api.Connector;
import ch.so.agi.gretl.logging.GretlLogger;
import ch.so.agi.gretl.logging.LogEnvironment;
import ch.so.agi.gretl.testutil.TestUtil;
import java.io.File;
import java.nio.file.Paths;

import static org.junit.Assert.assertEquals;

public class PostgisRasterExportStepTest {

Expand Down Expand Up @@ -66,7 +62,7 @@ public void export_geotiff_Ok() throws Exception {
@Test
public void export_aaigrid_Ok() throws Exception {
// Prepare
File sqlFile = new File("src/test/resources/data/postgisrasterprocessor/prepare_raster_aaigrid.sql");
File sqlFile = TestUtil.getResourceFile("data/postgisrasterprocessor/prepare_raster_aaigrid.sql");
String outFileName = "outfile.asc";

// TODO this file is missing in the resources, what should be actually tested here?
Expand Down
2 changes: 0 additions & 2 deletions gretl/src/test/java/ch/so/agi/gretl/steps/PublisherStepDb2LocalTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -474,7 +474,6 @@ public void db_regionsRegEx_UserFormats_AV() throws Exception {
}
}

// TODO - probably make a private class which will hold some of the stuff like connection and then just execute individual methods? Will reduce list of params at least...
private void deleteOutputFolder(Path targetFolder) throws IOException {
Objects.requireNonNull(targetFolder);
if (Files.exists(targetFolder)) {
Expand All @@ -490,7 +489,6 @@ private void initConfig(String datasetName, String basketHandlingType, List<Stri
PgMain pgMain = new PgMain();
pgMain.initConfig(config);

// FIXME
if (filePaths != null && !filePaths.isEmpty()) {
Path resultPath = resolvePath(SRC_TEST_DATA, filePaths);
config.setXtffile(resultPath.toString());
Expand Down
84 changes: 43 additions & 41 deletions gretl/src/test/java/ch/so/agi/gretl/steps/PublisherStepFile2RemoteTest.java
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,10 @@
package ch.so.agi.gretl.steps;

import ch.so.agi.gretl.testutil.SftpTest;
import com.github.robtimus.filesystems.sftp.SFTPEnvironment;
import org.junit.BeforeClass;
import org.junit.experimental.categories.Category;

import java.io.File;
import java.io.IOException;
import java.net.URI;
Expand All @@ -8,55 +13,52 @@
import java.nio.file.FileSystems;
import java.nio.file.Path;

import org.junit.Assert;
import org.junit.experimental.categories.Category;
@Category(SftpTest.class)
public class PublisherStepFile2RemoteTest extends AbstractPublisherStepTest {
private static final String FTP_URL = System.getProperty("ftpurl");
private static final String FTP_USER = System.getProperty("ftpusr");
private static final String FTP_PASSWORD = System.getProperty("ftppwd");
private static final String KNOWN_HOSTS_PATH = System.getProperty("user.home") + "/.ssh/known_hosts";

import com.github.robtimus.filesystems.sftp.SFTPEnvironment;
private static FileSystem fileSystem = null;
private static String path;

import ch.so.agi.gretl.testutil.SftpTest;
@BeforeClass
public static void initFileSystem() {
if (fileSystem != null) {
return;
}

try {
URI rawUri = new URI(FTP_URL);
path = rawUri.getRawPath();
URI host = createHostURI(rawUri);

@Category(SftpTest.class)
public class PublisherStepFile2RemoteTest extends AbstractPublisherStepTest {
private static FileSystem fileSystem=null;
static String ftpurl=System.getProperty("ftpurl");
static String ftpusr=System.getProperty("ftpusr");
static String ftppwd=System.getProperty("ftppwd");
static String path=null;
@org.junit.BeforeClass
static public void initFileSystem()
{
if(fileSystem==null) {
URI host=null;
URI rawuri=null;
try {
rawuri = new URI( ftpurl);
path=rawuri.getRawPath();
if(rawuri.getPort()==-1) {
host= new URI(rawuri.getScheme()+"://"+rawuri.getHost());
}else {
host= new URI(rawuri.getScheme()+"://"+rawuri.getHost()+":"+rawuri.getPort());
}
} catch (URISyntaxException e) {
throw new IllegalArgumentException(e);
}
SFTPEnvironment environment = new SFTPEnvironment()
.withUsername(ftpusr)
.withPassword(ftppwd.toCharArray())
.withKnownHosts(new File(System.getProperty("user.home"),".ssh/known_hosts"));
try {
fileSystem = FileSystems.newFileSystem( host, environment );
} catch (IOException e) {
throw new IllegalArgumentException(e);
}
.withUsername(FTP_USER)
.withPassword(FTP_PASSWORD.toCharArray())
.withKnownHosts(new File(KNOWN_HOSTS_PATH));

fileSystem = FileSystems.newFileSystem(host, environment);
} catch (URISyntaxException | IOException e) {
throw new IllegalArgumentException(e);
}
}
public PublisherStepFile2RemoteTest() {
super();

private static URI createHostURI(URI rawUri) throws URISyntaxException {
String scheme = rawUri.getScheme();
String host = rawUri.getHost();
int port = rawUri.getPort();

if (port == -1) {
return new URI(String.format("%s://%s", scheme, host));
} else {
return new URI(String.format("%s://%s:%d", scheme, host, port));
}
}

@Override
protected Path getTargetPath() {

Path remoteTestOut = fileSystem.getPath(path);
return remoteTestOut;
return fileSystem.getPath(path);
}
}
45 changes: 8 additions & 37 deletions gretl/src/test/java/ch/so/agi/gretl/steps/S3Bucket2BucketStepTest.java
View file
Original file line number Diff line number Diff line change
@@ -1,15 +1,12 @@
package ch.so.agi.gretl.steps;

import ch.so.agi.gretl.testutil.S3Test;
import ch.so.agi.gretl.testutil.S3TestHelper;
import ch.so.agi.gretl.testutil.TestUtil;
import org.junit.Rule;
import org.junit.Test;
import org.junit.experimental.categories.Category;
import org.junit.rules.TemporaryFolder;
import software.amazon.awssdk.auth.credentials.AwsBasicCredentials;
import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider;
import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider;
import software.amazon.awssdk.regions.Region;
import software.amazon.awssdk.services.s3.S3Client;
import software.amazon.awssdk.services.s3.model.DeleteObjectRequest;
import software.amazon.awssdk.services.s3.model.ListObjectsRequest;
Expand All @@ -19,8 +16,6 @@
import java.io.File;
import java.io.FileNotFoundException;
import java.io.UnsupportedEncodingException;
import java.net.URI;
import java.net.URISyntaxException;
import java.util.*;

import static org.junit.Assert.assertEquals;
Expand All @@ -34,6 +29,7 @@ public class S3Bucket2BucketStepTest {
private final String s3Endpoint;
private final String s3Region;
private final String acl;
private final S3TestHelper s3TestHelper;

@Rule
public TemporaryFolder folder = new TemporaryFolder();
Expand All @@ -46,22 +42,23 @@ public S3Bucket2BucketStepTest() {
this.s3Endpoint = "https://s3.eu-central-1.amazonaws.com";
this.s3Region = "eu-central-1";
this.acl = "public-read";
this.s3TestHelper = new S3TestHelper(this.s3AccessKey, this.s3SecretKey, this.s3Region, this.s3Endpoint);
}

@Test
@Category(S3Test.class)
public void copyFiles_Ok() throws Exception {
File sourceObject = TestUtil.getResourceFile(TestUtil.S3_BUCKET_DIR_PATH);
Map<String,String> metadata = new HashMap<>();
S3Client s3client = getS3Client();
S3Client s3Client = s3TestHelper.getS3Client();

deleteObjects(s3client, Arrays.asList("foo.txt", "bar.txt", "download.txt"));
uploadFiles(sourceObject, metadata);
deleteObjects(s3Client, Arrays.asList("foo.txt", "bar.txt", "download.txt"));
s3TestHelper.upload(sourceObject, metadata, s3SourceBucketName, acl);
copyFiles(metadata);

// Check result.
ListObjectsRequest listObjects = ListObjectsRequest.builder().bucket(s3TargetBucketName).build();
ListObjectsResponse res = s3client.listObjects(listObjects);
ListObjectsResponse res = s3Client.listObjects(listObjects);
List<S3Object> objects = res.contents();

List<String> keyList = new ArrayList<>();
Expand All @@ -73,23 +70,7 @@ public void copyFiles_Ok() throws Exception {
assertTrue(keyList.contains("bar.txt"));
assertTrue(keyList.contains("download.txt"));
assertEquals(3, keyList.size());
deleteFiles(s3client);
}

private S3Client getS3Client() {
return S3Client.builder()
.credentialsProvider(getCredentialsProvider())
.region(getRegion())
.endpointOverride(URI.create(s3Endpoint))
.build();
}

private AwsCredentialsProvider getCredentialsProvider() {
return StaticCredentialsProvider.create(AwsBasicCredentials.create(s3AccessKey, s3SecretKey));
}

private Region getRegion() {
return Region.of(s3Region);
deleteFiles(s3Client);
}

/**
Expand All @@ -111,16 +92,6 @@ private void deleteObjects(S3Client s3Client, List<String> keys) {
}
}

/**
* Upload files from a directory
* @param sourceObject directory for S3 bucket data on the local system
* @param metadata metadata
*/
private void uploadFiles(File sourceObject, Map<String, String> metadata) throws FileNotFoundException, URISyntaxException {
S3UploadStep s3UploadStep = new S3UploadStep();
s3UploadStep.execute(s3AccessKey, s3SecretKey, sourceObject, s3SourceBucketName, s3Endpoint, s3Region, acl, null, metadata);
}

/**
* Copy files from one bucket to another
* @param metadata metadata
Expand Down
Loading

0 comments on commit cc2fe0f

Please sign in to comment.