diff --git a/.editorconfig b/.editorconfig index c1322dc..1d13179 100644 --- a/.editorconfig +++ b/.editorconfig @@ -9,4 +9,7 @@ indent_size = 4 end_of_line = lf charset = utf-8 trim_trailing_whitespace = false -insert_final_newline = false \ No newline at end of file +insert_final_newline = false + +[build.gradle] +indent_style = tab diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml new file mode 100644 index 0000000..d49709c --- /dev/null +++ b/.github/workflows/check.yml @@ -0,0 +1,22 @@ +name: Check + +on: + pull_request: + branches: + - master + +jobs: + check: + runs-on: ubuntu-latest + steps: + - name: Checkout sources + uses: actions/checkout@v4 + + - name: Setup Java + uses: actions/setup-java@v3 + with: + distribution: temurin + java-version: 17 + + - name: Run check + run: ./gradlew check diff --git a/.vscode/settings.json b/.vscode/settings.json index 54fc50d..0f2452a 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -12,5 +12,6 @@ }, "[gradle]": { "editor.defaultFormatter": "richardwillis.vscode-spotless-gradle" - } + }, + "java.compile.nullAnalysis.mode": "automatic" } \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 0000000..9731614 --- /dev/null +++ b/README.md @@ -0,0 +1,36 @@ +# hale-transformer-api + +Spring Boot application that runs hale transformations based on requests read from a AMQP message queue. +The message queue needs to be provided externally, e.g. by a RabbitMQ instance. + +## Build + +To build the application, run + + ./gradlew build + +To start an instance, run + + ./gradlew bootRun + +## RabbitMQ + +To start a local RabbitMQ instance that can be used for debugging purposes, run + + docker compose up + +in the root directory of this project. + +The local instance can be also used to send messages to the queue that are then +processed by the application. When the application is running, an example message +like the following can be sent to the queue: + +```bash +$ docker exec -ti hale-transformer-api-rabbitmq-1 bash +root@123456789abc:/# rabbitmqadmin publish \ +> exchange=hale-transformer-exchange \ +> routing_key=hale.transformation.foo \ +> properties='{"content_type":"application/json"}' payload='{"projectUrl": "http://example.org/proj", "sourceDataUrl": "http://example.org/data"}' +Message published +root@123456789abc:/# +``` diff --git a/build.gradle b/build.gradle index e93df50..8a8c91d 100644 --- a/build.gradle +++ b/build.gradle @@ -9,23 +9,63 @@ group = 'to.wetransform.hale' version = '0.0.1-SNAPSHOT' java { - sourceCompatibility = '17' + sourceCompatibility = JavaVersion.VERSION_17 + targetCompatibility = JavaVersion.VERSION_17 } repositories { + // mavenLocal() //XXX for testing + maven { + // wetransform release repository (hale releases and Eclipse dependencies) + url 'https://artifactory.wetransform.to/artifactory/local' + } + // this needs to be defined before jcenter/MavenCentral for retrieving JAI + maven { + url 'https://repo.osgeo.org/repository/release/' + } mavenCentral() } +project.ext { + haleVersion = '5.1.0-SNAPSHOT' + cliVersion = '5.1.0-SNAPSHOT' + groovyVersion = '2.5.19' +} + dependencies { + // Spring implementation 'org.springframework.boot:spring-boot-starter-actuator' implementation 'org.springframework.boot:spring-boot-starter-quartz' implementation 'org.springframework.boot:spring-boot-starter-web' - implementation 'com.github.docker-java:docker-java-core:3.3.3' - implementation 'com.github.docker-java:docker-java-transport-httpclient5:3.3.3' + implementation 'org.springframework.boot:spring-boot-starter-amqp' implementation 'org.apache.httpcomponents.client5:httpclient5:5.2.1' + // hale + implementation 'eu.esdihumboldt.unpuzzled:org.eclipse.equinox.nonosgi.registry:1.0.0' + implementation "to.wetransform:hale-cli:$cliVersion", { + /* + * XXX The dependencies introduced by the schematron bundle cause some problems. + */ + exclude group: 'eu.esdihumboldt.hale', module: 'eu.esdihumboldt.hale.io.schematron' + } + + implementation "eu.esdihumboldt.hale:eu.esdihumboldt.hale.app.cli.commands:$haleVersion" + implementation "org.codehaus.groovy:groovy-all:$groovyVersion" + implementation 'org.json:json:20190722' + implementation 'org.slf4j:jul-to-slf4j:1.7.21' + implementation 'org.apache.httpcomponents:httpmime:4.5.14' + //developmentOnly 'org.springframework.boot:spring-boot-docker-compose' testImplementation 'org.springframework.boot:spring-boot-starter-test' + + // Logging + testRuntimeOnly 'ch.qos.logback:logback-core:1.4.11' + testRuntimeOnly 'ch.qos.logback:logback-classic:1.4.11' +} + +configurations.all { + // ensure SNAPSHOTs are updated every time if needed + resolutionStrategy.cacheChangingModulesFor 0, 'seconds' } tasks.named('test') { @@ -34,13 +74,23 @@ tasks.named('test') { spotless { java { - importOrder('groovy', 'java', 'javax', '') + palantirJavaFormat() + importOrder('java', 'javax', '') + removeUnusedImports() indentWithSpaces(4) trimTrailingWhitespace() endWithNewline() + + target 'src/*/java/**/*.java' } groovyGradle { target '*.gradle' // default target of groovyGradle - greclipse() } } + +/* + * Gradle wrapper + */ +wrapper { + gradleVersion = '8.4' +} diff --git a/compose.yaml b/compose.yaml index 0baad47..0f4b851 100644 --- a/compose.yaml +++ b/compose.yaml @@ -1 +1,6 @@ services: + rabbitmq: + image: rabbitmq:management + ports: + - "5672:5672" + - "15672:15672" diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index ac72c34..3fa8f86 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -1,6 +1,6 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-8.3-bin.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-8.4-bin.zip networkTimeout=10000 validateDistributionUrl=true zipStoreBase=GRADLE_USER_HOME diff --git a/src/main/java/to/wetransform/hale/transformer/CustomTarget.java b/src/main/java/to/wetransform/hale/transformer/CustomTarget.java new file mode 100644 index 0000000..63c1f8d --- /dev/null +++ b/src/main/java/to/wetransform/hale/transformer/CustomTarget.java @@ -0,0 +1,13 @@ +package to.wetransform.hale.transformer; + +import java.util.HashMap; +import java.util.Map; + +import eu.esdihumboldt.hale.common.core.io.Value; + +public record CustomTarget(String providerId, Map settings) { + + public CustomTarget(String providerId) { + this(providerId, new HashMap<>()); + } +} diff --git a/src/main/java/to/wetransform/hale/transformer/SourceConfig.java b/src/main/java/to/wetransform/hale/transformer/SourceConfig.java new file mode 100644 index 0000000..927bae2 --- /dev/null +++ b/src/main/java/to/wetransform/hale/transformer/SourceConfig.java @@ -0,0 +1,17 @@ +package to.wetransform.hale.transformer; + +import java.net.URI; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import eu.esdihumboldt.hale.common.core.io.Value; + +public record SourceConfig( + URI location, String providerId, Map settings, boolean transform, List attachments) { + + public SourceConfig(URI location, String providerId) { + this(location, providerId, new HashMap<>(), true, new ArrayList<>()); + } +} diff --git a/src/main/java/to/wetransform/hale/transformer/TargetConfig.java b/src/main/java/to/wetransform/hale/transformer/TargetConfig.java new file mode 100644 index 0000000..38a6548 --- /dev/null +++ b/src/main/java/to/wetransform/hale/transformer/TargetConfig.java @@ -0,0 +1,3 @@ +package to.wetransform.hale.transformer; + +public record TargetConfig(String filename, String preset, CustomTarget customTarget) {} diff --git a/src/main/java/to/wetransform/hale/transformer/Transformer.java b/src/main/java/to/wetransform/hale/transformer/Transformer.java new file mode 100644 index 0000000..4f1465b --- /dev/null +++ b/src/main/java/to/wetransform/hale/transformer/Transformer.java @@ -0,0 +1,80 @@ +package to.wetransform.hale.transformer; + +import java.io.InputStream; +import java.net.URI; +import java.text.MessageFormat; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; + +import eu.esdihumboldt.hale.app.transform.ExecContext; +import eu.esdihumboldt.hale.common.core.HalePlatform; +import eu.esdihumboldt.hale.common.core.io.project.model.Project; +import eu.esdihumboldt.hale.common.core.io.supplier.DefaultInputSupplier; +import eu.esdihumboldt.util.io.IOUtils; +import org.osgi.framework.Version; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import to.wetransform.halecli.internal.Init; + +public class Transformer { + + private static final Logger LOG = LoggerFactory.getLogger(Transformer.class); + + private CountDownLatch latch = new CountDownLatch(1); + + public void transform(/* TODO add parameters for data and project sources */ ) { + // TODO setup log files for reports and transformation log + + long heapMaxSize = Runtime.getRuntime().maxMemory(); + LOG.info("Maximum heap size configured as " + IOUtils.humanReadableByteCount(heapMaxSize, false)); + + Init.init(); + + Version version = HalePlatform.getCoreVersion(); + LOG.info(MessageFormat.format("Launching hale-transformer {0}...", version.toString())); + + ExecContext context = new ExecContext(); + + // URI projectUri = .... + // context.setProject(projectUri); + // Project project = loadProject(projectUri); + + // context.setSources(...) + // context.setSourceProviderIds(...) + // context.setSourcesSettings(...) + + // Value sourceCrs = null; + // TODO determine source CRS + + // TargetConfig targetConfig = configureTarget(project, sourceCrs); + + try { + // run the transformation + + LOG.info("Transforming..."); + TimeUnit.SECONDS.sleep(30); + // new ExecTransformation().run(context); + + LOG.info("Transformation complete."); + } catch (Throwable t) { + LOG.error("Failed to execute transformation: " + t.getMessage(), t); + } finally { + latch.countDown(); + } + } + + private Project loadProject(URI projectUri) { + DefaultInputSupplier supplier = new DefaultInputSupplier(projectUri); + Project result = null; + try (InputStream in = supplier.getInput()) { + result = Project.load(in); + } catch (Exception e) { + LOG.warn("Could not load project file to determine presets: " + e.getStackTrace()); + } + return result; + } + + public CountDownLatch getLatch() { + return latch; + } +} diff --git a/src/main/java/to/wetransform/hale/transformer/TransformerConfig.java b/src/main/java/to/wetransform/hale/transformer/TransformerConfig.java new file mode 100644 index 0000000..e0b76a4 --- /dev/null +++ b/src/main/java/to/wetransform/hale/transformer/TransformerConfig.java @@ -0,0 +1,6 @@ +package to.wetransform.hale.transformer; + +public class TransformerConfig { + + // empty for now +} diff --git a/src/main/java/to/wetransform/hale/transformer/api/TransformationController.java b/src/main/java/to/wetransform/hale/transformer/api/TransformationController.java deleted file mode 100644 index 7164611..0000000 --- a/src/main/java/to/wetransform/hale/transformer/api/TransformationController.java +++ /dev/null @@ -1,56 +0,0 @@ -package to.wetransform.hale.transformer.api; - -import com.github.dockerjava.api.DockerClient; -import com.github.dockerjava.api.command.LogContainerCmd; -import com.github.dockerjava.api.command.WaitContainerResultCallback; -import com.github.dockerjava.core.DefaultDockerClientConfig; -import com.github.dockerjava.core.DockerClientConfig; -import com.github.dockerjava.core.DockerClientImpl; -import com.github.dockerjava.httpclient5.ApacheDockerHttpClient; -import com.github.dockerjava.transport.DockerHttpClient; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.web.bind.annotation.GetMapping; -import org.springframework.web.bind.annotation.RestController; - -@RestController -public class TransformationController { - - private static final Logger log = LoggerFactory.getLogger(TransformationController.class); - - private static final String TRANSFORMER_IMAGE = "wetransform/hale-transformer:latest"; // TODO Should be configurable - - // TODO Adapt endpoints to OGC API Processes - @GetMapping("/transform") - public void transform() { - String containerId; - WaitContainerResultCallback callback; - - DockerClientConfig std = DefaultDockerClientConfig.createDefaultConfigBuilder().build(); - - DockerHttpClient httpClient = new ApacheDockerHttpClient.Builder() - .dockerHost(std.getDockerHost()) - .build(); - - DockerClient docker = DockerClientImpl.getInstance(std, httpClient); - - // TODO Implement container start here - // Data and project source must be configurable (in first step as URLs) - // If transformation project get baked into the Docker image later on, configuration could - // be done e.g. via an identifier - - // Dummy Docker operation "getId" -> replace with logic to run transformer image - containerId = docker.createContainerCmd(TRANSFORMER_IMAGE) - .withEnv("HALE_OPTS=-Dlog.hale.level=INFO -Dlog.root.level=WARN -Xmx800m", - "HT_PROJECT_URL=https://wetransform.box.com/shared/static/pvtiecxvpuuo061t7mrmuu8iknrj84oj.halez", - "HT_SOURCE_URL=https://wetransform.box.com/shared/static/gub5gnfv7wljekwwglo33on58a2w6gk3.gml") - .exec().getId(); - - LogContainerCmd logCmd = docker.logContainerCmd(containerId); - logCmd.withStdOut(true).withStdErr(true).withTimestamps(true); - - log.info(docker.inspectImageCmd(TRANSFORMER_IMAGE).exec().getCreated()); - - docker.startContainerCmd(containerId).exec(); - } -} diff --git a/src/main/java/to/wetransform/hale/transformer/api/TransformerApiApplication.java b/src/main/java/to/wetransform/hale/transformer/api/TransformerApiApplication.java index de290d8..d8c6d80 100644 --- a/src/main/java/to/wetransform/hale/transformer/api/TransformerApiApplication.java +++ b/src/main/java/to/wetransform/hale/transformer/api/TransformerApiApplication.java @@ -1,13 +1,63 @@ package to.wetransform.hale.transformer.api; +import org.springframework.amqp.core.Binding; +import org.springframework.amqp.core.BindingBuilder; +import org.springframework.amqp.core.Queue; +import org.springframework.amqp.core.TopicExchange; +import org.springframework.amqp.rabbit.connection.ConnectionFactory; +import org.springframework.amqp.rabbit.core.RabbitTemplate; +import org.springframework.amqp.support.converter.Jackson2JsonMessageConverter; +import org.springframework.amqp.support.converter.MessageConverter; import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.context.annotation.Bean; +import org.springframework.scheduling.annotation.EnableScheduling; @SpringBootApplication +@EnableScheduling public class TransformerApiApplication { + // TODO Should be configurable + private static final String ROUTING_KEY = "hale.transformation.#"; + + // TODO Should be configurable + public static final String TOPIC_EXCHANGE_NAME = "hale-transformer-exchange"; + + // TODO Should be configurable + public static final String QUEUE_NAME = "hale-transformation"; + + @Bean + Queue queue() { + // TODO Queue should be declared passively, i.e. it should be created + // outside of this application + return new Queue(QUEUE_NAME, false); + } + + @Bean + TopicExchange exchange() { + // TODO Exchange should be declared passively, i.e. it should be created + // outside of this application + return new TopicExchange(TOPIC_EXCHANGE_NAME); + } + + @Bean + Binding binding(Queue queue, TopicExchange exchange) { + return BindingBuilder.bind(queue).to(exchange).with(ROUTING_KEY); + } + + @Bean + public MessageConverter jsonMessageConverter() { + return new Jackson2JsonMessageConverter(); + } + + @Bean + public RabbitTemplate rabbitTemplate(final ConnectionFactory connectionFactory) { + final var rabbitTemplate = new RabbitTemplate(connectionFactory); + rabbitTemplate.setMessageConverter(jsonMessageConverter()); + return rabbitTemplate; + } + public static void main(String[] args) { SpringApplication.run(TransformerApiApplication.class, args); } - } diff --git a/src/main/java/to/wetransform/hale/transformer/api/messaging/TransformationMessageConsumer.java b/src/main/java/to/wetransform/hale/transformer/api/messaging/TransformationMessageConsumer.java new file mode 100644 index 0000000..1f5d524 --- /dev/null +++ b/src/main/java/to/wetransform/hale/transformer/api/messaging/TransformationMessageConsumer.java @@ -0,0 +1,46 @@ +package to.wetransform.hale.transformer.api.messaging; + +import java.io.Serializable; +import java.util.concurrent.TimeUnit; + +import com.fasterxml.jackson.annotation.JsonProperty; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.amqp.rabbit.annotation.RabbitListener; +import org.springframework.stereotype.Service; +import to.wetransform.hale.transformer.Transformer; +import to.wetransform.hale.transformer.api.TransformerApiApplication; + +@Service +public class TransformationMessageConsumer { + /** + * + */ + public record TransformationMessage( + @JsonProperty("projectUrl") String projectUrl, @JsonProperty("sourceDataUrl") String sourceDataUrl) + implements Serializable {} + + private static final Logger LOG = LoggerFactory.getLogger(TransformationMessageConsumer.class); + + @RabbitListener(queues = TransformerApiApplication.QUEUE_NAME) + public void receiveMessage(final TransformationMessage message) { + LOG.info("Received projectUrl = " + message.projectUrl + " sourceDataUrl = " + message.sourceDataUrl); + + // TODO Implement mechanism to only accept a message from the queue if no + // transformation is currently running + + if (message.projectUrl != null && message.sourceDataUrl() != null) { + Transformer tx = new Transformer(); + + try { + tx.transform(); + tx.getLatch().await(10, TimeUnit.MINUTES); // TODO make configurable + } catch (InterruptedException e) { + // TODO What should be done when the transformation fails or times out? + // - Simply requeuing the message is probably not helpful + // - Send a message back so that the producer can react? + LOG.error("Transformation process timed out: " + e.getMessage(), e); + } + } + } +} diff --git a/src/main/resources/logback.xml b/src/main/resources/logback.xml new file mode 100644 index 0000000..841f9c9 --- /dev/null +++ b/src/main/resources/logback.xml @@ -0,0 +1,11 @@ + + + + %d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n + + + + + + + diff --git a/src/test/java/to/wetransform/hale/transformer/api/TransformerApiApplicationTests.java b/src/test/java/to/wetransform/hale/transformer/api/TransformerApiApplicationTests.java index f1e5996..a1191f6 100644 --- a/src/test/java/to/wetransform/hale/transformer/api/TransformerApiApplicationTests.java +++ b/src/test/java/to/wetransform/hale/transformer/api/TransformerApiApplicationTests.java @@ -7,7 +7,5 @@ class TransformerApiApplicationTests { @Test - void contextLoads() { - } - + void contextLoads() {} }