From 4a887c7b3cd5e640b724d0d9e8688a47f07c8333 Mon Sep 17 00:00:00 2001 From: Emanuela Epure <67077116+emanuelaepure10@users.noreply.github.com> Date: Fri, 15 Dec 2023 16:06:53 +0100 Subject: [PATCH] Update Transformer.java update --- build.gradle | 4 +- .../hale/transformer/SourceConfig.java | 10 +- .../hale/transformer/TargetConfig.java | 3 +- .../hale/transformer/Transformer.java | 378 ++++++++++-------- .../api/TransformerApiApplication.java | 86 ++-- 5 files changed, 265 insertions(+), 216 deletions(-) diff --git a/build.gradle b/build.gradle index 99b6d8c..8a8c91d 100644 --- a/build.gradle +++ b/build.gradle @@ -74,12 +74,12 @@ tasks.named('test') { spotless { java { - /*palantirJavaFormat() + palantirJavaFormat() importOrder('java', 'javax', '') removeUnusedImports() indentWithSpaces(4) trimTrailingWhitespace() - endWithNewline()*/ + endWithNewline() target 'src/*/java/**/*.java' } diff --git a/src/main/java/to/wetransform/hale/transformer/SourceConfig.java b/src/main/java/to/wetransform/hale/transformer/SourceConfig.java index 4bf2dff..927bae2 100644 --- a/src/main/java/to/wetransform/hale/transformer/SourceConfig.java +++ b/src/main/java/to/wetransform/hale/transformer/SourceConfig.java @@ -8,10 +8,10 @@ import eu.esdihumboldt.hale.common.core.io.Value; -public record SourceConfig(URI location, String providerId, Map settings, boolean transform, - List attachments) { +public record SourceConfig( + URI location, String providerId, Map settings, boolean transform, List attachments) { - public SourceConfig(URI location, String providerId) { - this(location, providerId, new HashMap<>(), true, new ArrayList<>()); - } + public SourceConfig(URI location, String providerId) { + this(location, providerId, new HashMap<>(), true, new ArrayList<>()); + } } diff --git a/src/main/java/to/wetransform/hale/transformer/TargetConfig.java b/src/main/java/to/wetransform/hale/transformer/TargetConfig.java index 986fcb8..38a6548 100644 --- a/src/main/java/to/wetransform/hale/transformer/TargetConfig.java +++ b/src/main/java/to/wetransform/hale/transformer/TargetConfig.java @@ -1,4 +1,3 @@ package to.wetransform.hale.transformer; -public record TargetConfig(String filename, String preset, CustomTarget customTarget) { -} +public record TargetConfig(String filename, String preset, CustomTarget customTarget) {} diff --git a/src/main/java/to/wetransform/hale/transformer/Transformer.java b/src/main/java/to/wetransform/hale/transformer/Transformer.java index 9d2d0a9..3846faf 100644 --- a/src/main/java/to/wetransform/hale/transformer/Transformer.java +++ b/src/main/java/to/wetransform/hale/transformer/Transformer.java @@ -4,20 +4,20 @@ import java.net.URI; import java.nio.file.Files; import java.text.MessageFormat; -import java.util.HashMap; -import java.util.List; -import java.util.Map; +import java.util.*; import java.util.Map.Entry; -import java.util.Optional; import java.util.concurrent.CountDownLatch; +import java.util.stream.Collectors; import com.google.common.base.Strings; import eu.esdihumboldt.hale.app.transform.ExecContext; import eu.esdihumboldt.hale.app.transform.ExecTransformation; import eu.esdihumboldt.hale.common.core.HalePlatform; import eu.esdihumboldt.hale.common.core.io.HaleIO; +import eu.esdihumboldt.hale.common.core.io.IOProvider; import eu.esdihumboldt.hale.common.core.io.Value; import eu.esdihumboldt.hale.common.core.io.extension.IOProviderDescriptor; +import eu.esdihumboldt.hale.common.core.io.extension.IOProviderExtension; import eu.esdihumboldt.hale.common.core.io.project.model.IOConfiguration; import eu.esdihumboldt.hale.common.core.io.project.model.Project; import eu.esdihumboldt.hale.common.core.io.supplier.DefaultInputSupplier; @@ -29,6 +29,7 @@ import org.apache.commons.io.LineIterator; import org.apache.commons.io.output.TeeOutputStream; import org.apache.commons.lang.exception.ExceptionUtils; +import org.eclipse.core.runtime.content.IContentType; import org.json.JSONObject; import org.osgi.framework.Version; import org.slf4j.Logger; @@ -66,70 +67,6 @@ public Transformer(TransformerConfig config) { this.config = config; } - public static String processLog( - LineIterator it, int maxLogSizeCharacters, int maxEndLines, int maxEndLinesCharacters) { - StringBuilder logBuilder = new StringBuilder(); - - boolean truncated = false; - - while (it.hasNext() && !truncated && logBuilder.length() < maxLogSizeCharacters) { - String line = it.nextLine(); - - int left = maxLogSizeCharacters - logBuilder.length(); - if (left >= line.length()) { - logBuilder.append(line); - logBuilder.append('\n'); - } else { - if (left > MESSAGE_LINE_TRUNCATED.length()) { - logBuilder.append(line, 0, left - MESSAGE_LINE_TRUNCATED.length()); - logBuilder.append(MESSAGE_LINE_TRUNCATED); - logBuilder.append('\n'); - } - truncated = true; - } - } - - if (it.hasNext()) { - truncated = true; - } - - if (truncated) { - // attempt to add lines from the end of the log - CircularFifoBuffer endLines = new CircularFifoBuffer(maxEndLines); - boolean overflow = false; - while (it.hasNext()) { - if (endLines.size() == maxEndLines) { - overflow = true; - } - endLines.add(it.nextLine()); - } - - int endSize = endLines.stream().mapToInt(l -> l.toString().length()).sum(); - if (endSize > maxEndLinesCharacters) { - // don't add end lines - logBuilder.append(MESSAGE_LOG_TRUNCATED); - logBuilder.append('\n'); - } else { - if (!overflow) { - // nothing was truncated -> just add the lines - } else { - logBuilder.append("[Log truncated... following are the last "); - logBuilder.append(endLines.size()); - logBuilder.append(" lines of the log]"); - logBuilder.append('\n'); - } - - // add end lines - for (Object line : endLines) { - logBuilder.append(line.toString()); - logBuilder.append('\n'); - } - } - } - - return logBuilder.toString(); - } - public void transform(String sourceDataURL, String projectURL, String targetURL) { boolean success = false; PrintStream sysOut = System.out; @@ -212,35 +149,18 @@ public void transform(String sourceDataURL, String projectURL, String targetURL) context.setSourceProviderIds(sourceProviderIds); // Set up custom defaultSrs - List> sourcesSettings = List.of(Map.of("defaultSrs", Value.of("something"))); - context.setSourcesSettings(sourcesSettings); + // TODO change this to something real + List> sourceConfigs = List.of(Map.of("defaultSrs", Value.of("something"))); + context.setSourcesSettings(sourceConfigs); // extract detected source data crs and use in target config Value sourceCrs = null; - // if (!sourceConfigs.isEmpty()) { - // Throwable error = null; - // // try each source config, in case it is not set for some - // for (int i = 0; i < sourceConfigs.size() && (sourceCrs == null || sourceCrs.isEmpty()); i++) { - // try { - // sourceCrs = sourceConfigs.get(i).getSettings().get("defaultSrs"); - // } catch (Throwable t) { - // error = t; - // } - // } - // - // if (error != null) { - // log.warn("Could not determine source data CRS", error); - // } - // else if (sourceCrs == null || sourceCrs.isEmpty()) { - // log.warn("Unable to determine source data CRS: None of {} sources is configured with a CRS", - // sourceConfigs.size()); - // } - // } - // else { - // log.warn("Unable to determine source data CRS: No source configured"); - // } - - // TargetConfig targetConfig = configureTarget(project, sourceCrs); + // try each source config, in case it is not set for some + for (int i = 0; i < sourceConfigs.size() && (sourceCrs == null || sourceCrs.isEmpty()); i++) { + sourceCrs = sourceConfigs.get(i).get("defaultSrs"); + } + + TargetConfig targetConfig = configureTarget(project, sourceCrs); // run the transformation LOG.info("Transforming..."); @@ -296,6 +216,70 @@ private void performCallback( } } + public static String processLog( + LineIterator it, int maxLogSizeCharacters, int maxEndLines, int maxEndLinesCharacters) { + StringBuilder logBuilder = new StringBuilder(); + + boolean truncated = false; + + while (it.hasNext() && !truncated && logBuilder.length() < maxLogSizeCharacters) { + String line = it.nextLine(); + + int left = maxLogSizeCharacters - logBuilder.length(); + if (left >= line.length()) { + logBuilder.append(line); + logBuilder.append('\n'); + } else { + if (left > MESSAGE_LINE_TRUNCATED.length()) { + logBuilder.append(line, 0, left - MESSAGE_LINE_TRUNCATED.length()); + logBuilder.append(MESSAGE_LINE_TRUNCATED); + logBuilder.append('\n'); + } + truncated = true; + } + } + + if (it.hasNext()) { + truncated = true; + } + + if (truncated) { + // attempt to add lines from the end of the log + CircularFifoBuffer endLines = new CircularFifoBuffer(maxEndLines); + boolean overflow = false; + while (it.hasNext()) { + if (endLines.size() == maxEndLines) { + overflow = true; + } + endLines.add(it.nextLine()); + } + + int endSize = endLines.stream().mapToInt(l -> l.toString().length()).sum(); + if (endSize > maxEndLinesCharacters) { + // don't add end lines + logBuilder.append(MESSAGE_LOG_TRUNCATED); + logBuilder.append('\n'); + } else { + if (!overflow) { + // nothing was truncated -> just add the lines + } else { + logBuilder.append("[Log truncated... following are the last "); + logBuilder.append(endLines.size()); + logBuilder.append(" lines of the log]"); + logBuilder.append('\n'); + } + + // add end lines + for (Object line : endLines) { + logBuilder.append(line.toString()); + logBuilder.append('\n'); + } + } + } + + return logBuilder.toString(); + } + private Project loadProject(URI projectUri) { DefaultInputSupplier supplier = new DefaultInputSupplier(projectUri); Project result = null; @@ -307,57 +291,118 @@ private Project loadProject(URI projectUri) { return result; } - // private TargetConfig configureTarget(Project lp, Value sourceCrs) { - // Map presets = getPresets(lp); - - // // create a target configuration - // //FIXME for now this is a fixed configuration - - // TargetConfig result = new TargetConfig(lp, sourceCrs, null); - - // String defaultPreset = "default"; - // String hcPreset = "hale-connect"; - - // if (presets.containsKey(hcPreset)) { - // // project contains hale connect preset - // result.setPreset(hcPreset); - // IOConfiguration preset = presets.get(hcPreset); - // result.setFilename(determineTargetFileName(preset)); - // } - // else if (presets.containsKey(defaultPreset)) { - // // project contains default preset - // result.setPreset(defaultPreset); - // IOConfiguration preset = presets.get(defaultPreset); - // result.setFilename(determineTargetFileName(preset)); - // } - // else { - // CustomTarget target = new CustomTarget(); - - // // WFS 2 FeatureCollection - // // target.setProviderId("eu.esdihumboldt.hale.io.wfs.fc.write-2.0"); - // // GML FeatureCollection (to make bsp happy) - // target.setProviderId("eu.esdihumboldt.hale.io.gml.writer"); - - // // for testing: - // target.getSettings().put("xml.pretty", Value.of(true)); - // target.getSettings().put("crs.epsg.prefix", - // Value.of("http://www.opengis.net/def/crs/EPSG/0/")); - - // // use crs from source data analysis if available and a valid epsg code, - // otherwise fallback to EPSG:4326 - // Value targetCrs = (sourceCrs != null && - // sourceCrs.getStringRepresentation().startsWith("code:EPSG")) ? sourceCrs : - // Value.of("code:EPSG:4326"); - // target.getSettings().put("crs", targetCrs); - // LOG.info("Using " + targetCrs.getStringRepresentation() + " as transformation - // target crs"); - - // result.setFilename("inspire.gml"); - // result.setCustomTarget(target); - // } - - // return result; - // } + private TargetConfig configureTarget(Project lp, Value sourceCrs) { + String filename; + String preset = null; + CustomTarget customTarget = null; + + Map presets = getPresets(lp); + + // Preset names + String defaultPreset = "default"; + String hcPreset = "hale-connect"; + + if (presets.containsKey(hcPreset)) { + // Project contains hale-connect preset + preset = hcPreset; + IOConfiguration ioConfiguration = presets.get(hcPreset); + filename = determineTargetFileName(ioConfiguration); + } else if (presets.containsKey(defaultPreset)) { + // Project contains default preset + preset = defaultPreset; + IOConfiguration ioConfiguration = presets.get(defaultPreset); + filename = determineTargetFileName(ioConfiguration); + } else { + // No specific presets found, creating a custom target configuration + + Map targetMap = new HashMap<>(); + + // Specify target provider for GML FeatureCollection + String targetProvider = "eu.esdihumboldt.hale.io.gml.writer"; + + // Additional settings for testing + targetMap.put("xml.pretty", Value.of(true)); + targetMap.put("crs.epsg.prefix", Value.of("http://www.opengis.net/def/crs/EPSG/0/")); + + // Use CRS from source data analysis if available and a valid EPSG code, + // otherwise fallback to EPSG:4326 + Value targetCrs = + (sourceCrs != null && sourceCrs.getStringRepresentation().startsWith("code:EPSG")) + ? sourceCrs + : Value.of("code:EPSG:4326"); + + targetMap.put("crs", targetCrs); + LOG.info("Using {} as the transformation target CRS", targetCrs.getStringRepresentation()); + + // Create a custom target configuration + CustomTarget target = new CustomTarget(targetProvider, targetMap); + + filename = "inspire.gml"; + customTarget = target; + } + + // Create and return the target configuration + return new TargetConfig(filename, preset, customTarget); + } + + /** + * Determine the name of the target file based on an export preset. + * + * @param preset the export preset + * @return the file name for the target file + */ + public static String determineTargetFileName(IOConfiguration preset) { + // Default extension to "xml" to reflect old behavior + String extension = "xml"; + + IContentType contentType = determineContentType(preset); + if (contentType != null) { + // Derive extension from content type + String[] extensions = contentType.getFileSpecs(IContentType.FILE_EXTENSION_SPEC); + if (extensions != null && extensions.length > 0) { + extension = extensions[0]; // Choose the first one + } + } + + // If extension would be "gml," use "xml" instead for backward compatibility + extension = "gml".equalsIgnoreCase(extension) ? "xml" : extension; + + LOG.info("Chose .{} as the extension for the target file", extension); + + return "result." + extension; + } + + private static IContentType determineContentType(IOConfiguration preset) { + // Usually, the content type is part of the settings + Value value = preset.getProviderConfiguration().get(IOProvider.PARAM_CONTENT_TYPE); + if (value != null && !value.isEmpty()) { + return HalePlatform.getContentTypeManager().getContentType(value.as(String.class)); + } + + // Try to determine based on provider ID + String providerId = preset.getProviderId(); + if (providerId != null) { + IOProviderDescriptor providerDescriptor = + IOProviderExtension.getInstance().getFactory(providerId); + if (providerDescriptor != null) { + Set supportedTypes = providerDescriptor.getSupportedTypes(); + if (!supportedTypes.isEmpty()) { + IContentType contentType = supportedTypes.iterator().next(); + + if (supportedTypes.size() > 1) { + LOG.warn( + "Multiple content types as candidates ({}), chose {}", + supportedTypes.stream().map(IContentType::getId).collect(Collectors.joining(", ")), + contentType.getId()); + } + + return contentType; + } + } + } + + return null; + } /** * Get all export presets from the project. @@ -367,35 +412,40 @@ private Project loadProject(URI projectUri) { */ private Map getPresets(Project project) { Map exportPresets = new HashMap<>(); + if (project == null) { return exportPresets; } - for (Entry preset : + for (Entry entry : project.getExportConfigurations().entrySet()) { - IOConfiguration conf = preset.getValue(); - if (InstanceIO.ACTION_SAVE_TRANSFORMED_DATA.equals(conf.getActionId())) { - // configuration for data export - IOConfiguration c = conf.clone(); - String name = preset.getKey(); - - // check provider - IOProviderDescriptor factory = - HaleIO.findIOProviderFactory(InstanceWriter.class, null, c.getProviderId()); - if (factory != null) { - if (Strings.isNullOrEmpty(name)) { - name = factory.getDisplayName(); - } - exportPresets.put(name, c); - } else { - LOG.error(MessageFormat.format( - "I/O provider {1} for export preset {0} not found", name, c.getProviderId())); - } + IOConfiguration originalConfiguration = entry.getValue(); + + if (InstanceIO.ACTION_SAVE_TRANSFORMED_DATA.equals(originalConfiguration.getActionId())) { + String presetName = entry.getKey(); + IOConfiguration clonedConfiguration = originalConfiguration.clone(); + + // Check and add the I/O provider to exportPresets + checkAndAddIOProvider(presetName, clonedConfiguration, exportPresets); } } + return exportPresets; } + private void checkAndAddIOProvider( + String presetName, IOConfiguration configuration, Map exportPresets) { + String providerId = configuration.getProviderId(); + IOProviderDescriptor factory = HaleIO.findIOProviderFactory(InstanceWriter.class, null, providerId); + + if (factory != null) { + String name = Strings.isNullOrEmpty(presetName) ? factory.getDisplayName() : presetName; + exportPresets.computeIfAbsent(name, k -> configuration); + } else { + LOG.error("I/O provider {} for export preset {} not found", providerId, presetName); + } + } + public CountDownLatch getLatch() { return latch; } diff --git a/src/main/java/to/wetransform/hale/transformer/api/TransformerApiApplication.java b/src/main/java/to/wetransform/hale/transformer/api/TransformerApiApplication.java index 244febb..d8c6d80 100644 --- a/src/main/java/to/wetransform/hale/transformer/api/TransformerApiApplication.java +++ b/src/main/java/to/wetransform/hale/transformer/api/TransformerApiApplication.java @@ -17,47 +17,47 @@ @EnableScheduling public class TransformerApiApplication { - // TODO Should be configurable - private static final String ROUTING_KEY = "hale.transformation.#"; - - // TODO Should be configurable - public static final String TOPIC_EXCHANGE_NAME = "hale-transformer-exchange"; - - // TODO Should be configurable - public static final String QUEUE_NAME = "hale-transformation"; - - @Bean - Queue queue() { - // TODO Queue should be declared passively, i.e. it should be created - // outside of this application - return new Queue(QUEUE_NAME, false); - } - - @Bean - TopicExchange exchange() { - // TODO Exchange should be declared passively, i.e. it should be created - // outside of this application - return new TopicExchange(TOPIC_EXCHANGE_NAME); - } - - @Bean - Binding binding(Queue queue, TopicExchange exchange) { - return BindingBuilder.bind(queue).to(exchange).with(ROUTING_KEY); - } - - @Bean - public MessageConverter jsonMessageConverter() { - return new Jackson2JsonMessageConverter(); - } - - @Bean - public RabbitTemplate rabbitTemplate(final ConnectionFactory connectionFactory) { - final var rabbitTemplate = new RabbitTemplate(connectionFactory); - rabbitTemplate.setMessageConverter(jsonMessageConverter()); - return rabbitTemplate; - } - - public static void main(String[] args) { - SpringApplication.run(TransformerApiApplication.class, args); - } + // TODO Should be configurable + private static final String ROUTING_KEY = "hale.transformation.#"; + + // TODO Should be configurable + public static final String TOPIC_EXCHANGE_NAME = "hale-transformer-exchange"; + + // TODO Should be configurable + public static final String QUEUE_NAME = "hale-transformation"; + + @Bean + Queue queue() { + // TODO Queue should be declared passively, i.e. it should be created + // outside of this application + return new Queue(QUEUE_NAME, false); + } + + @Bean + TopicExchange exchange() { + // TODO Exchange should be declared passively, i.e. it should be created + // outside of this application + return new TopicExchange(TOPIC_EXCHANGE_NAME); + } + + @Bean + Binding binding(Queue queue, TopicExchange exchange) { + return BindingBuilder.bind(queue).to(exchange).with(ROUTING_KEY); + } + + @Bean + public MessageConverter jsonMessageConverter() { + return new Jackson2JsonMessageConverter(); + } + + @Bean + public RabbitTemplate rabbitTemplate(final ConnectionFactory connectionFactory) { + final var rabbitTemplate = new RabbitTemplate(connectionFactory); + rabbitTemplate.setMessageConverter(jsonMessageConverter()); + return rabbitTemplate; + } + + public static void main(String[] args) { + SpringApplication.run(TransformerApiApplication.class, args); + } }