Skip to content

Commit

Permalink
wip: homogenisierung testcode
Browse files Browse the repository at this point in the history
  • Loading branch information
@dkubanyi
dkubanyi authored and Daniel Kubanyi committed Jun 6, 2024
1 parent d82b5ff commit 8b144ea
Show file tree
Hide file tree
Showing 6 changed files with 119 additions and 136 deletions.
6 changes: 0 additions & 6 deletions gretl/src/test/java/ch/so/agi/gretl/steps/Csv2ExcelStepTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,12 +39,6 @@ public class Csv2ExcelStepTest {

@Rule
public TemporaryFolder folder = new TemporaryFolder();

private GretlLogger log;

public Csv2ExcelStepTest() {
this.log = LogEnvironment.getLogger(this.getClass());
}

@Test
public void encoding_iso_8859_1_Ok() throws Exception {
Expand Down
190 changes: 97 additions & 93 deletions gretl/src/test/java/ch/so/agi/gretl/steps/Csv2ParquetStepTest.java
View file
Original file line number Diff line number Diff line change
@@ -1,9 +1,5 @@
package ch.so.agi.gretl.steps;

import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;

import java.nio.file.Path;
import java.nio.file.Paths;
import java.time.Instant;
Expand All @@ -14,6 +10,7 @@
import java.time.ZoneOffset;
import java.time.format.DateTimeFormatter;

import ch.so.agi.gretl.testutil.TestUtil;
import org.apache.avro.generic.GenericRecord;
import org.apache.commons.io.FilenameUtils;
import org.apache.hadoop.conf.Configuration;
Expand All @@ -31,19 +28,13 @@
import ch.so.agi.gretl.logging.GretlLogger;
import ch.so.agi.gretl.logging.LogEnvironment;

import static org.junit.Assert.*;

public class Csv2ParquetStepTest {

private static final Configuration testConf = new Configuration();
@Rule
public TemporaryFolder folder = new TemporaryFolder();

private GretlLogger log;

private static final Configuration testConf = new Configuration();

public Csv2ParquetStepTest() {
this.log = LogEnvironment.getLogger(this.getClass());
}

@Test
public void encoding_iso_8859_1_Ok() throws Exception {
// Prepare
Expand All @@ -64,11 +55,13 @@ public void encoding_iso_8859_1_Ok() throws Exception {
// Validate
org.apache.hadoop.fs.Path resultFile = new org.apache.hadoop.fs.Path(Paths
.get(outputPath.toString(), FilenameUtils.getBaseName(csvPath.toString()) + ".parquet").toString());
ParquetReader<GenericRecord> reader = AvroParquetReader
.<GenericRecord>builder(HadoopInputFile.fromPath(resultFile, testConf)).build();

GenericRecord arecord = reader.read();
assertEquals("Fernwärme", arecord.get("Energietraeger").toString());
try (ParquetReader<GenericRecord> reader = AvroParquetReader
.<GenericRecord>builder(HadoopInputFile.fromPath(resultFile, testConf)).build()
) {
GenericRecord arecord = reader.read();
assertEquals("Fernwärme", arecord.get("Energietraeger").toString());
}
}

@Test
Expand All @@ -90,27 +83,29 @@ public void date_datatypes_handling_Ok() throws Exception {
// Validate
org.apache.hadoop.fs.Path resultFile = new org.apache.hadoop.fs.Path(Paths
.get(outputPath.toString(), FilenameUtils.getBaseName(csvPath.toString()) + ".parquet").toString());
ParquetReader<GenericRecord> reader = AvroParquetReader
.<GenericRecord>builder(HadoopInputFile.fromPath(resultFile, testConf)).build();

GenericRecord arecord = reader.read();
assertEquals("1", arecord.get("id").toString());
try (ParquetReader<GenericRecord> reader = AvroParquetReader
.<GenericRecord>builder(HadoopInputFile.fromPath(resultFile, testConf)).build()
) {
GenericRecord arecord = reader.read();
assertEquals("1", arecord.get("id").toString());

LocalDate resultDate = Instant.ofEpochSecond((int)arecord.get("aDate") * 24 * 60 * 60).atZone(ZoneId.systemDefault()).toLocalDate();
LocalDate expectedDate = LocalDate.parse("2023-06-24", DateTimeFormatter.ISO_LOCAL_DATE);
assertEquals(expectedDate, resultDate);
LocalDate resultDate = Instant.ofEpochSecond((int)arecord.get("aDate") * 24 * 60 * 60).atZone(ZoneId.systemDefault()).toLocalDate();
LocalDate expectedDate = LocalDate.parse("2023-06-24", DateTimeFormatter.ISO_LOCAL_DATE);
assertEquals(expectedDate, resultDate);

// FIXME Falls parquet-avro fix i.O. (auch atZone muss nach systemDefault geändert werden).
LocalDateTime resultDateTime = Instant.ofEpochMilli((long)arecord.get("aDatetime")).atZone(ZoneOffset.UTC).toLocalDateTime();
LocalDateTime expectedDateTime = LocalDateTime.parse("1977-09-16T04:46:12"); // Das "lokale" Datum wird nach UTC beim Schreiben umgewandelt. Ist kompliziert... Siehe Code.
assertEquals(expectedDateTime, resultDateTime);
// FIXME Falls parquet-avro fix i.O. (auch atZone muss nach systemDefault geändert werden).
LocalDateTime resultDateTime = Instant.ofEpochMilli((long)arecord.get("aDatetime")).atZone(ZoneOffset.UTC).toLocalDateTime();
LocalDateTime expectedDateTime = LocalDateTime.parse("1977-09-16T04:46:12"); // Das "lokale" Datum wird nach UTC beim Schreiben umgewandelt. Ist kompliziert... Siehe Code.
assertEquals(expectedDateTime, resultDateTime);

LocalTime resultTime = Instant.ofEpochMilli(((Number)arecord.get("aTime")).longValue()).atZone(ZoneOffset.UTC).toLocalTime();
LocalTime expectedTime = LocalTime.parse("15:51:41"); // Siehe datetime. Aber noch kein Bugfix vorhanden. Geht vielleicht gar nicht, weil AVRO nur Time _ohne_ Zone kennt.
assertEquals(expectedTime, resultTime);
LocalTime resultTime = Instant.ofEpochMilli(((Number)arecord.get("aTime")).longValue()).atZone(ZoneOffset.UTC).toLocalTime();
LocalTime expectedTime = LocalTime.parse("15:51:41"); // Siehe datetime. Aber noch kein Bugfix vorhanden. Geht vielleicht gar nicht, weil AVRO nur Time _ohne_ Zone kennt.
assertEquals(expectedTime, resultTime);

GenericRecord nextRecord = reader.read();
assertNull(nextRecord);
GenericRecord nextRecord = reader.read();
assertNull(nextRecord);
}
}

@Test
Expand All @@ -132,28 +127,31 @@ public void model_set_Ok() throws Exception {
// Validate
org.apache.hadoop.fs.Path resultFile = new org.apache.hadoop.fs.Path(Paths
.get(outputPath.toString(), FilenameUtils.getBaseName(csvPath.toString()) + ".parquet").toString());
ParquetReader<GenericRecord> reader = AvroParquetReader
.<GenericRecord>builder(HadoopInputFile.fromPath(resultFile, testConf)).build();

int recordCount = 0;
GenericRecord arecord = reader.read();
GenericRecord firstRecord = null;
GenericRecord lastRecord = null;
while(arecord != null) {
if (recordCount == 0) {
firstRecord = arecord;

try (ParquetReader<GenericRecord> reader = AvroParquetReader
.<GenericRecord>builder(HadoopInputFile.fromPath(resultFile, testConf)).build()
) {
int recordCount = 0;
GenericRecord arecord = reader.read();
GenericRecord firstRecord = null;
GenericRecord lastRecord = null;
while(arecord != null) {
if (recordCount == 0) {
firstRecord = arecord;
}
recordCount++;

lastRecord = arecord;
arecord = reader.read();
}
recordCount++;

lastRecord = arecord;
arecord = reader.read();

assertEquals(31, recordCount);
assertNotNull(firstRecord);
assertEquals(1991, firstRecord.get("jahr"));
assertNull(firstRecord.get("internet_clicks_durchschnitt_pro_monat"));
assertEquals(2021, lastRecord.get("jahr"));
assertEquals(999, lastRecord.get("internet_clicks_durchschnitt_pro_monat"));
}

assertEquals(31, recordCount);
assertEquals(1991, firstRecord.get("jahr"));
assertEquals(null, firstRecord.get("internet_clicks_durchschnitt_pro_monat"));
assertEquals(2021, lastRecord.get("jahr"));
assertEquals(999, lastRecord.get("internet_clicks_durchschnitt_pro_monat"));
}

@Test
Expand All @@ -164,7 +162,7 @@ public void no_model_set_Ok() throws Exception {
settings.setValue(IoxWkfConfig.SETTING_VALUEDELIMITER, null);
settings.setValue(IoxWkfConfig.SETTING_VALUESEPARATOR, ";");

Path csvPath = Paths.get("src/test/resources/data/csv2parquet/bewilligte_erdwaermeanlagen/bewilligte_erdwaermeanlagen_excel_export.csv");
Path csvPath = TestUtil.getResourceFile("data/csv2parquet/bewilligte_erdwaermeanlagen/bewilligte_erdwaermeanlagen_excel_export.csv").toPath();
Path outputPath = folder.newFolder().toPath();

// Run
Expand All @@ -174,27 +172,30 @@ public void no_model_set_Ok() throws Exception {
// Validate
org.apache.hadoop.fs.Path resultFile = new org.apache.hadoop.fs.Path(Paths
.get(outputPath.toString(), FilenameUtils.getBaseName(csvPath.toString()) + ".parquet").toString());
ParquetReader<GenericRecord> reader = AvroParquetReader
.<GenericRecord>builder(HadoopInputFile.fromPath(resultFile, testConf)).build();

int recordCount = 0;
GenericRecord arecord = reader.read();
GenericRecord firstRecord = null;
GenericRecord lastRecord = null;
while(arecord != null) {
if (recordCount == 0) {
firstRecord = arecord;

try (ParquetReader<GenericRecord> reader = AvroParquetReader
.<GenericRecord>builder(HadoopInputFile.fromPath(resultFile, testConf)).build()
) {
int recordCount = 0;
GenericRecord arecord = reader.read();
GenericRecord firstRecord = null;
GenericRecord lastRecord = null;
while(arecord != null) {
if (recordCount == 0) {
firstRecord = arecord;
}
recordCount++;

lastRecord = arecord;
arecord = reader.read();
}
recordCount++;

lastRecord = arecord;
arecord = reader.read();

assertEquals(31, recordCount);
assertNotNull(firstRecord);
assertTrue(firstRecord.hasField("durchschnittlicher_oelpreis_pro_1000_liter"));
assertTrue((firstRecord.get("durchschnittlicher_oelpreis_pro_1000_liter") == null));
assertEquals("999", (lastRecord.get("internet_clicks_durchschnitt_pro_monat").toString()));
}

assertEquals(31, recordCount);
assertTrue(firstRecord.hasField("durchschnittlicher_oelpreis_pro_1000_liter"));
assertTrue((firstRecord.get("durchschnittlicher_oelpreis_pro_1000_liter") == null));
assertEquals("999", (lastRecord.get("internet_clicks_durchschnitt_pro_monat").toString()));
}

@Test
Expand All @@ -205,7 +206,7 @@ public void custom_config_and_no_model_set_Ok() throws Exception {
settings.setValue(IoxWkfConfig.SETTING_VALUEDELIMITER, "'");
settings.setValue(IoxWkfConfig.SETTING_VALUESEPARATOR, ";");

Path csvPath = Paths.get("src/test/resources/data/csv2parquet/bewilligte_erdwaermeanlagen/bewilligte_erdwaermeanlagen_semikolon_hochkomma.csv");
Path csvPath = TestUtil.getResourceFile("data/csv2parquet/bewilligte_erdwaermeanlagen/bewilligte_erdwaermeanlagen_semikolon_hochkomma.csv").toPath();
Path outputPath = folder.newFolder().toPath();

// Run
Expand All @@ -215,26 +216,29 @@ public void custom_config_and_no_model_set_Ok() throws Exception {
// Validate
org.apache.hadoop.fs.Path resultFile = new org.apache.hadoop.fs.Path(Paths
.get(outputPath.toString(), FilenameUtils.getBaseName(csvPath.toString()) + ".parquet").toString());
ParquetReader<GenericRecord> reader = AvroParquetReader
.<GenericRecord>builder(HadoopInputFile.fromPath(resultFile, testConf)).build();

int recordCount = 0;
GenericRecord arecord = reader.read();
GenericRecord firstRecord = null;
GenericRecord lastRecord = null;
while(arecord != null) {
if (recordCount == 0) {
firstRecord = arecord;

try (ParquetReader<GenericRecord> reader = AvroParquetReader
.<GenericRecord>builder(HadoopInputFile.fromPath(resultFile, testConf)).build()
) {
int recordCount = 0;
GenericRecord arecord = reader.read();
GenericRecord firstRecord = null;
GenericRecord lastRecord = null;
while(arecord != null) {
if (recordCount == 0) {
firstRecord = arecord;
}
recordCount++;

lastRecord = arecord;
arecord = reader.read();
}
recordCount++;

lastRecord = arecord;
arecord = reader.read();

assertEquals(31, recordCount);
assertNotNull(firstRecord);
assertTrue(firstRecord.hasField("durchschnittlicher_oelpreis_pro_1000_liter"));
assertTrue((firstRecord.get("durchschnittlicher_oelpreis_pro_1000_liter") == null));
assertEquals("999", (lastRecord.get("internet_clicks_durchschnitt_pro_monat").toString()));
}

assertEquals(31, recordCount);
assertTrue(firstRecord.hasField("durchschnittlicher_oelpreis_pro_1000_liter"));
assertTrue((firstRecord.get("durchschnittlicher_oelpreis_pro_1000_liter") == null));
assertEquals("999", (lastRecord.get("internet_clicks_durchschnitt_pro_monat").toString()));
}
}
21 changes: 8 additions & 13 deletions gretl/src/test/java/ch/so/agi/gretl/steps/DatabaseDocumentExportStepTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,22 +24,16 @@
public class DatabaseDocumentExportStepTest {

@ClassRule
public static PostgreSQLContainer postgres =
(PostgreSQLContainer) new PostgisContainerProvider()
.newInstance().withDatabaseName("gretl")
.withUsername(TestUtil.PG_DDLUSR_USR)
.withPassword(TestUtil.PG_DDLUSR_PWD)
.withInitScript("data/sql/init_postgresql.sql")
.waitingFor(Wait.forLogMessage(TestUtil.WAIT_PATTERN, 2));


public DatabaseDocumentExportStepTest() {
this.log = LogEnvironment.getLogger(this.getClass());
}
public static PostgreSQLContainer<?> postgres =
(PostgreSQLContainer<?>) new PostgisContainerProvider().newInstance()
.withDatabaseName(TestUtil.PG_DB_NAME)
.withUsername(TestUtil.PG_DDLUSR_USR)
.withPassword(TestUtil.PG_DDLUSR_PWD)
.withInitScript(TestUtil.PG_INIT_SCRIPT_PATH)
.waitingFor(Wait.forLogMessage(TestUtil.WAIT_PATTERN, 2));

@Rule
public TemporaryFolder folder = new TemporaryFolder();
private GretlLogger log;

// TODO:
// Die mühsamen self-signed Zertifikate des AIO können schlecht getestet werden. A) Nur intern B) Filenamen können sich ändern.
Expand All @@ -58,6 +52,7 @@ public void exportDocuments_Ok() throws Exception {

Connection con = connector.connect();
con.setAutoCommit(false);

try {
Statement stmt = con.createStatement();

Expand Down
17 changes: 3 additions & 14 deletions gretl/src/test/java/ch/so/agi/gretl/steps/Db2DbStepTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,18 +40,11 @@ public class Db2DbStepTest {
.withInitScript(TestUtil.PG_INIT_SCRIPT_PATH)
.waitingFor(Wait.forLogMessage(TestUtil.WAIT_PATTERN, 2));

private final GretlLogger log;
private Connector connector;

@Rule
public TemporaryFolder folder = new TemporaryFolder();

public Db2DbStepTest() {
LogEnvironment.initStandalone();
this.log = LogEnvironment.getLogger(this.getClass());
// this.connector = new Connector("jdbc:derby:memory:myInMemDB;create=true", "bjsvwsch", null);
}

@Before
public void initialize() throws Exception {
this.connector = new Connector("jdbc:derby:memory:myInMemDB;create=true", "bjsvwsch", null);
Expand All @@ -71,20 +64,18 @@ public void finalise() throws Exception {
public void faultFreeExecutionTest() throws Exception {
File sqlFile = TestUtil.createTempFile(folder, "SELECT * FROM colors; ", "query.sql");
File sqlFile2 = TestUtil.createTempFile(folder, "SELECT * FROM colors; ", "query2.sql");

ArrayList<TransferSet> transferSets = new ArrayList<>(Arrays.asList(
new TransferSet(sqlFile.getAbsolutePath(), "colors_copy", Boolean.TRUE),
new TransferSet(sqlFile2.getAbsolutePath(), "colors_copy", Boolean.TRUE)
));

Connector sourceDb = new Connector("jdbc:derby:memory:myInMemDB;create=true", "bjsvwsch", null);
Connector targetDb = new Connector("jdbc:derby:memory:myInMemDB;create=true", "bjsvwsch", null);
Db2DbStep db2db = new Db2DbStep();

db2db.processAllTransferSets(sourceDb, targetDb, transferSets);

try (Connection connection = this.connector.connect()) {
ResultSet rs = connection.createStatement()
.executeQuery("SELECT * FROM colors_copy WHERE farbname = 'blau'");
ResultSet rs = connection.createStatement().executeQuery("SELECT * FROM colors_copy WHERE farbname = 'blau'");
while (rs.next()) {
assertEquals(rs.getObject("rot"), 0);
assertEquals(rs.getObject("farbname"), "blau");
Expand Down Expand Up @@ -118,7 +109,7 @@ public void fileWithMultipleStmtTest() throws Exception {

@Test(expected = EmptyFileException.class)
public void db2dbEmptyFileTest() throws Exception {
File sqlFile = folder.newFile("query.sql");
File sqlFile = TestUtil.createTempFile(folder, "", "query.sql");
ArrayList<TransferSet> transferSets = new ArrayList<>(Collections.singletonList(
new TransferSet(sqlFile.getAbsolutePath(), "colors_copy", Boolean.FALSE)
));
Expand Down Expand Up @@ -655,7 +646,6 @@ private void createTestDb(Connector sourceDb) throws Exception {
Connection con = sourceDb.connect();
createTableInTestDb(con);
writeExampleDataInTestDB(con);

}

private void createTableInTestDb(Connection con) throws Exception {
Expand All @@ -672,5 +662,4 @@ private void writeExampleDataInTestDB(Connection con) throws Exception {
stmt.execute("INSERT INTO colors VALUES (251,0,0,'rot')");
stmt.execute("INSERT INTO colors VALUES (0,0,255,'blau')");
}

}
Loading

0 comments on commit 8b144ea

Please sign in to comment.