Skip to content

Commit

Permalink
wip: improve code quality
Browse files Browse the repository at this point in the history 
Signed-off-by: Daniel Kubányi <[email protected]>
  • Loading branch information
@dkubanyi
dkubanyi committed Jun 12, 2024
1 parent b01b490 commit 3b2e2aa
Show file tree
Hide file tree
Showing 11 changed files with 776 additions and 1,017 deletions.
450 changes: 219 additions & 231 deletions gretl/src/test/java/ch/so/agi/gretl/steps/AbstractPublisherStepTest.java
View file

Large diffs are not rendered by default.

36 changes: 9 additions & 27 deletions gretl/src/test/java/ch/so/agi/gretl/steps/Csv2ExcelStepTest.java
View file
Original file line number Diff line number Diff line change
@@ -1,28 +1,9 @@
package ch.so.agi.gretl.steps;

import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;

import java.io.File;
import java.io.FileInputStream;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.time.Instant;
import java.time.LocalDate;
import java.time.LocalDateTime;
import java.time.LocalTime;
import java.time.ZoneId;
import java.time.ZoneOffset;
import java.time.format.DateTimeFormatter;

import ch.ehi.basics.settings.Settings;
import ch.interlis.iom_j.csv.CsvReader;
import ch.interlis.ioxwkf.dbtools.IoxWkfConfig;
import ch.so.agi.gretl.testutil.TestUtil;
import org.apache.avro.generic.GenericRecord;
import org.apache.commons.io.FilenameUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.parquet.avro.AvroParquetReader;
import org.apache.parquet.hadoop.ParquetReader;
import org.apache.parquet.hadoop.util.HadoopInputFile;
import org.apache.poi.ss.usermodel.Row;
import org.apache.poi.xssf.usermodel.XSSFSheet;
import org.apache.poi.xssf.usermodel.XSSFWorkbook;
Expand All @@ -31,11 +12,12 @@
import org.junit.Test;
import org.junit.rules.TemporaryFolder;

import ch.ehi.basics.settings.Settings;
import ch.interlis.iom_j.csv.CsvReader;
import ch.interlis.ioxwkf.dbtools.IoxWkfConfig;
import ch.so.agi.gretl.logging.GretlLogger;
import ch.so.agi.gretl.logging.LogEnvironment;
import java.io.File;
import java.io.FileInputStream;
import java.nio.file.Path;
import java.nio.file.Paths;

import static org.junit.Assert.assertEquals;

public class Csv2ExcelStepTest {

Expand Down
63 changes: 29 additions & 34 deletions gretl/src/test/java/ch/so/agi/gretl/steps/DatabaseDocumentExportStepTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,8 +17,6 @@
import ch.so.agi.gretl.logging.LogEnvironment;
import ch.so.agi.gretl.testutil.TestUtil;

import javax.xml.crypto.Data;

import static org.junit.Assert.*;

public class DatabaseDocumentExportStepTest {
Expand All @@ -33,13 +31,19 @@ public class DatabaseDocumentExportStepTest {
.waitingFor(Wait.forLogMessage(TestUtil.WAIT_PATTERN, 2));

private final GretlLogger log;
private final String schemaName;
private final String tableName;
private final String columnName;
private Connector connector;

@Rule
public TemporaryFolder folder = new TemporaryFolder();

public DatabaseDocumentExportStepTest() {
this.log = LogEnvironment.getLogger(this.getClass());
this.schemaName = "ada_denkmalschutz";
this.tableName = "fachapplikation_rechtsvorschrift_link";
this.columnName = "multimedia_link";
}

// TODO:
Expand All @@ -61,21 +65,11 @@ public void after() throws Exception {

@Test
public void exportDocuments_Ok() throws Exception {
String schemaName = "ada_denkmalschutz";
String tableName = "fachapplikation_rechtsvorschrift_link";
String columnName = "multimedia_link";
File targetDir = folder.newFolder();

try (Connection con = connector.connect(); Statement stmt = con.createStatement()) {
con.setAutoCommit(false);

stmt.execute("DROP SCHEMA IF EXISTS "+schemaName+" CASCADE;");
stmt.execute("CREATE SCHEMA "+schemaName+";");
stmt.execute("CREATE TABLE "+schemaName+"."+tableName+" (id serial, "+columnName+" text);");
//https://artplus.verw.rootso.org/MpWeb-apSolothurnDenkmal/download/2W8v0qRZQBC0ahDnZGut3Q?mode=gis
//http://geo.so.ch/models/ilimodels.xml
//http://models.geo.admin.ch/ilimodels.xml

initializeSchema(con, stmt);
stmt.execute("INSERT INTO "+schemaName+"."+tableName+" ("+columnName+") VALUES('http://models.geo.admin.ch/ilimodels.xml');");
con.commit();

Expand All @@ -95,17 +89,11 @@ public void exportDocuments_Ok() throws Exception {
// https://stackoverflow.com/questions/1884230/httpurlconnection-doesnt-follow-redirect-from-http-to-https
@Test
public void exportDocuments_Fail() throws Exception {
String schemaName = "ada_denkmalschutz";
String tableName = "fachapplikation_rechtsvorschrift_link";
String columnName = "multimedia_link";
File targetDir = folder.newFolder();

try (Connection con = connector.connect(); Statement stmt = con.createStatement()) {
con.setAutoCommit(false);

stmt.execute("DROP SCHEMA IF EXISTS "+schemaName+" CASCADE;");
stmt.execute("CREATE SCHEMA "+schemaName+";");
stmt.execute("CREATE TABLE "+schemaName+"."+tableName+" (id serial, "+columnName+" text);");
initializeSchema(con, stmt);
stmt.execute("INSERT INTO "+schemaName+"."+tableName+" ("+columnName+") VALUES('https://geo.so.ch/fubar');");
con.commit();

Expand All @@ -119,19 +107,14 @@ public void exportDocuments_Fail() throws Exception {

@Test
public void exportDocuments_WithoutPrefix_Ok() throws Exception {
String schemaName = "ada_denkmalschutz";
String tableName = "fachapplikation_rechtsvorschrift_link";
String columnName = "multimedia_link";
File targetDir = folder.newFolder();

log.debug("targetDir: " + targetDir.getAbsolutePath());

try (Connection con = connector.connect(); Statement stmt = con.createStatement()) {
con.setAutoCommit(false);
initializeSchema(con, stmt);

stmt.execute("DROP SCHEMA IF EXISTS "+schemaName+" CASCADE;");
stmt.execute("CREATE SCHEMA "+schemaName+";");
stmt.execute("CREATE TABLE "+schemaName+"."+tableName+" (id serial, "+columnName+" text);");
stmt.execute("INSERT INTO "+schemaName+"."+tableName+" ("+columnName+") VALUES('http://models.geo.admin.ch/ilimodels.xml');");
con.commit();

Expand All @@ -149,19 +132,12 @@ public void exportDocuments_WithoutPrefix_Ok() throws Exception {

@Test
public void exportDocuments_WithoutFileNameExtension_Ok() throws Exception {
String schemaName = "ada_denkmalschutz";
String tableName = "fachapplikation_rechtsvorschrift_link";
String columnName = "multimedia_link";

File targetDir = folder.newFolder();
log.debug("targetDir: " + targetDir.getAbsolutePath());

try (Connection con = connector.connect(); Statement stmt = con.createStatement()) {
con.setAutoCommit(false);

stmt.execute("DROP SCHEMA IF EXISTS "+schemaName+" CASCADE;");
stmt.execute("CREATE SCHEMA "+schemaName+";");
stmt.execute("CREATE TABLE "+schemaName+"."+tableName+" (id serial, "+columnName+" text);");
initializeSchema(con, stmt);
stmt.execute("INSERT INTO "+schemaName+"."+tableName+" ("+columnName+") VALUES('http://models.geo.admin.ch/ilimodels.xml');");
con.commit();

Expand All @@ -176,4 +152,23 @@ public void exportDocuments_WithoutFileNameExtension_Ok() throws Exception {
assertTrue(content.contains("IliRepository"));
}
}

/**
* Creates a schema and facghapplikation table to test against
* @param connection the db connection
* @param statement the statement
* @throws Exception if execution fails
*
* <ul>
* <li><a href="https://artplus.verw.rootso.org/MpWeb-apSolothurnDenkmal/download/2W8v0qRZQBC0ahDnZGut3Q?mode=gis">Docs</a></li>
* <li><a href="http://geo.so.ch/models/ilimodels.xml">Docs</a></li>
* <li><a href="http://models.geo.admin.ch/ilimodels.xml">Docs</a></li>
* </ul>
*/
private void initializeSchema(Connection connection, Statement statement) throws Exception {
statement.execute("DROP SCHEMA IF EXISTS " + this.schemaName + " CASCADE;");
statement.execute("CREATE SCHEMA " + this.schemaName + ";");
statement.execute("CREATE TABLE " + this.schemaName + "." + this.tableName + " (id serial, " + this.columnName + " text);");
connection.commit();
}
}
16 changes: 2 additions & 14 deletions gretl/src/test/java/ch/so/agi/gretl/steps/Gpkg2ShpStepTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,16 +23,6 @@ public class Gpkg2ShpStepTest {
@Rule
public TemporaryFolder folder = new TemporaryFolder();

// @Test
// public void dummy() throws Exception {
// String TEST_OUT = "/Users/stefan/tmp/shapefix/";
// File gpkgFile = new File("src/test/resources/data/gpkg2shp/verkehrszaehlstellen_edit.gpkg");
//
// Gpkg2ShpStep gpkg2shpStep = new Gpkg2ShpStep();
// gpkg2shpStep.execute(gpkgFile.getAbsolutePath(), TEST_OUT);
//
// }

@Test
public void export_no_Geometry_Ok() throws Exception {
String TEST_OUT = folder.newFolder().getAbsolutePath();
Expand All @@ -54,8 +44,7 @@ public void export_no_Geometry_Ok() throws Exception {
@Test
public void export_Ok() throws Exception {
String TEST_OUT = folder.newFolder().getAbsolutePath();
File gpkgFile = new File("src/test/resources/data/gpkg2shp/ch.so.agi_av_gb_administrative_einteilungen_2020-08-20.gpkg");

File gpkgFile = TestUtil.getResourceFile("data/gpkg2shp/ch.so.agi_av_gb_administrative_einteilungen_2020-08-20.gpkg");
Gpkg2ShpStep gpkg2shpStep = new Gpkg2ShpStep();
gpkg2shpStep.execute(gpkgFile.getAbsolutePath(), TEST_OUT);

Expand Down Expand Up @@ -84,8 +73,7 @@ public void export_more_attribute_types_Ok() throws Exception {
String TEST_OUT = folder.newFolder().getAbsolutePath();
//String TEST_OUT = "/Users/stefan/tmp/shp/";

File gpkgFile = new File("src/test/resources/data/gpkg2shp/ch.so.afu.abbaustellen.gpkg");

File gpkgFile = TestUtil.getResourceFile("data/gpkg2shp/ch.so.afu.abbaustellen.gpkg");
Gpkg2ShpStep gpkg2shpStep = new Gpkg2ShpStep();
gpkg2shpStep.execute(gpkgFile.getAbsolutePath(), TEST_OUT);

Expand Down
8 changes: 4 additions & 4 deletions gretl/src/test/java/ch/so/agi/gretl/steps/GzipStepTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,15 +19,15 @@
import static org.junit.Assert.assertTrue;

public class GzipStepTest {

public GzipStepTest() {
this.log = LogEnvironment.getLogger(this.getClass());
}

private final GretlLogger log;
@Rule
public TemporaryFolder folder = new TemporaryFolder();

public GzipStepTest() {
this.log = LogEnvironment.getLogger(this.getClass());
}

@Test
public void compress_file_Ok() throws Exception {
File outDirectory = folder.newFolder("transformFile_Ok");
Expand Down
41 changes: 19 additions & 22 deletions gretl/src/test/java/ch/so/agi/gretl/steps/JsonImportStepTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,11 +30,20 @@ public class JsonImportStepTest {
.withInitScript(TestUtil.PG_INIT_SCRIPT_PATH)
.waitingFor(Wait.forLogMessage(TestUtil.WAIT_PATTERN, 2));

private final String schemaName;
private final String tableName;
private final String columnName;
private Connector connector;

@Rule
public TemporaryFolder folder = new TemporaryFolder();

public JsonImportStepTest() {
this.schemaName = "jsonimport";
this.tableName = "jsonobject";
this.columnName = "json_text_col";
}

@Before
public void before() throws Exception {
this.connector = new Connector(postgres.getJdbcUrl(), TestUtil.PG_DDLUSR_USR, TestUtil.PG_DDLUSR_PWD);
Expand All @@ -50,16 +59,10 @@ public void after() throws Exception {
@Test
public void importJsonObject_Ok() throws Exception {
File jsonFile = TestUtil.createTempFile(folder, "{\"foo\":\"bar\"}", "test1.json");
String schemaName = "jsonimport";
String tableName = "jsonobject";
String columnName = "json_text_col";

try (Connection con = connector.connect(); Statement stmt = con.createStatement()) {
con.setAutoCommit(false);
stmt.execute("DROP SCHEMA IF EXISTS "+schemaName+" CASCADE;");
stmt.execute("CREATE SCHEMA "+schemaName+";");
stmt.execute("CREATE TABLE "+schemaName+"."+tableName+" (id serial, "+columnName+" text);");
con.commit();
initializeSchema(con, stmt);

JsonImportStep jsonImportStep = new JsonImportStep();
jsonImportStep.execute(connector, jsonFile, schemaName+"."+tableName, columnName, true);
Expand All @@ -77,17 +80,10 @@ public void importJsonObject_Ok() throws Exception {
@Test
public void appendJsonObject_Ok() throws Exception {
File jsonFile = TestUtil.createTempFile(folder, "{\"foo\":\"bar\"}", "test2.json");
String schemaName = "jsonimport";
String tableName = "jsonobject";
String columnName = "json_text_col";

try (Connection con = connector.connect(); Statement stmt = con.createStatement()) {
con.setAutoCommit(false);
stmt.execute("DROP SCHEMA IF EXISTS "+schemaName+" CASCADE;");
stmt.execute("CREATE SCHEMA "+schemaName+";");
stmt.execute("CREATE TABLE "+schemaName+"."+tableName+" (id serial, "+columnName+" text);");
stmt.execute("INSERT INTO "+schemaName+"."+tableName+" ("+columnName+") VALUES ('{\"yin\": \"yang\"}');");
con.commit();
initializeSchema(con, stmt);

JsonImportStep jsonImportStep = new JsonImportStep();
jsonImportStep.execute(connector, jsonFile, schemaName+"."+tableName, columnName, false);
Expand All @@ -103,16 +99,10 @@ public void appendJsonObject_Ok() throws Exception {
@Test
public void importJsonArray_Ok() throws Exception {
File jsonFile = TestUtil.createTempFile(folder, "[{\"type\":\"building\"}, {\"type\":\"street\"}]", "test3.json");
String schemaName = "jsonimport";
String tableName = "jsonarray";
String columnName = "json_text_col";

try (Connection con = connector.connect(); Statement stmt = con.createStatement()) {
con.setAutoCommit(false);
stmt.execute("DROP SCHEMA IF EXISTS "+schemaName+" CASCADE;");
stmt.execute("CREATE SCHEMA "+schemaName+";");
stmt.execute("CREATE TABLE "+schemaName+"."+tableName+" (id serial, "+columnName+" text);");
con.commit();
initializeSchema(con, stmt);

JsonImportStep jsonImportStep = new JsonImportStep();
jsonImportStep.execute(connector, jsonFile, schemaName+"."+tableName, columnName, true);
Expand All @@ -130,4 +120,11 @@ public void importJsonArray_Ok() throws Exception {
assertFalse(rs.next());
}
}

private void initializeSchema(Connection connection, Statement statement) throws Exception {
statement.execute("DROP SCHEMA IF EXISTS " + this.schemaName + " CASCADE;");
statement.execute("CREATE SCHEMA " + this.schemaName+";");
statement.execute("CREATE TABLE " + this.schemaName + "."+this.tableName + " (id serial, " +this.columnName + " text);");
connection.commit();
}
}
Loading

0 comments on commit 3b2e2aa

Please sign in to comment.