From 4afaf652c647be624bf4191fc73123e55d75c805 Mon Sep 17 00:00:00 2001 From: Dominik Riemer Date: Tue, 3 Oct 2023 20:39:27 +0200 Subject: [PATCH 1/4] refactor(#1960): Refactor transformation rules, add schema restore --- .../AdapterEventPreviewPipeline.java | 3 +- streampipes-connect-shared/pom.xml | 10 ++ .../shared/AdapterPipelineGeneratorBase.java | 95 ++---------- .../SupportsNestedTransformationRule.java | 56 +++++++ .../convert/ProvidesConversionResult.java | 11 +- .../convert/SchemaConverter.java | 67 +++++++++ .../convert/ToOriginalSchemaConverter.java | 141 ++++++++++++++++++ .../convert/ToTransformedSchemaConverter.java | 140 +++++++++++++++++ ...AdapterTransformationPipelineElement.java} | 23 ++- .../elements/AddTimestampPipelineElement.java | 39 ----- ...TransformSchemaAdapterPipelineElement.java | 81 ---------- .../TransformStreamAdapterElement.java | 71 --------- .../TransformValueAdapterPipelineElement.java | 96 ------------ ...fulTransformationRuleGeneratorVisitor.java | 99 ++++++++++++ ...essTransformationRuleGeneratorVisitor.java | 134 +++++++++++++++++ .../TransformationRuleGeneratorVisitor.java} | 18 ++- .../transform/TransformationRule.java | 2 +- ...e.java => AddValueTransformationRule.java} | 19 ++- .../CreateNestedTransformationRule.java | 31 ++-- .../schema/DeleteTransformationRule.java | 29 ++-- .../schema/MoveTransformationRule.java | 18 +-- .../schema/RenameTransformationRule.java | 36 ++--- .../schema/SchemaEventTransformer.java | 119 --------------- .../DuplicateFilterPipelineElement.java | 8 +- .../stream/EventRateTransformationRule.java | 10 +- .../stream/StreamEventTransformer.java | 66 -------- ...va => AddTimestampTransformationRule.java} | 16 +- .../CorrectionValueTransformationRule.java | 68 ++++----- .../value/DatatypeTransformationRule.java | 18 +-- .../value/TimestampTranformationRuleMode.java | 13 +- ....java => TimestampTransformationRule.java} | 60 +++----- .../value/UnitTransformationRule.java | 59 ++++---- .../value/ValueEventTransformer.java | 83 ----------- .../preprocessing/utils/ConversionUtils.java | 93 ++++++++++++ .../{Util.java => utils/Utils.java} | 16 +- .../shared/preprocessing/UtilsTest.java | 45 ++++++ .../shared/preprocessing/convert/Helpers.java | 92 ++++++++++++ .../ToOriginalSchemaConverterTest.java | 104 +++++++++++++ .../ToTransformedSchemaConverterTest.java | 121 +++++++++++++++ ...pterTransformationPipelineElementTest.java | 60 ++++++++ .../CreateNestedTransformationRuleTest.java | 4 +- .../schema/DeleteTransformationRuleTest.java | 4 +- .../schema/MoveTransformationRuleTest.java | 6 +- .../schema/RenameTransformationRuleTest.java | 4 +- .../schema/SchemaEventTransformerTest.java | 9 +- .../transform/value/CorrectionValueTest.java | 14 +- .../value/UnitTransformRuleTest.java | 10 +- .../value/ValueEventTransformerTest.java | 10 +- .../adapter/AdapterPipelineGenerator.java | 32 ++-- .../generic/elements/AddTimestampTest.java | 6 +- .../generic/elements/DuplicateFilterTest.java | 34 ++--- .../rules/ITransformationRuleVisitor.java | 59 ++++++++ .../rules/TransformationRuleDescription.java | 4 + .../schema/CreateNestedRuleDescription.java | 12 ++ .../rules/schema/DeleteRuleDescription.java | 12 ++ .../rules/schema/MoveRuleDescription.java | 12 ++ .../rules/schema/RenameRuleDescription.java | 12 ++ ...ventRateTransformationRuleDescription.java | 12 ++ ...plicatesTransformationRuleDescription.java | 12 ++ .../value/AddTimestampRuleDescription.java | 12 ++ ...AddValueTransformationRuleDescription.java | 12 ++ ...DatatypeTransformationRuleDescription.java | 12 ++ ...ionValueTransformationRuleDescription.java | 12 ++ ...mestampTranfsformationRuleDescription.java | 12 ++ .../TimestampTransformationRuleMode.java | 26 ---- .../value/UnitTransformRuleDescription.java | 12 ++ .../event-property-row.component.ts | 25 ++-- .../event-schema/event-schema.component.ts | 1 + .../edit-schema-transformation.component.html | 2 +- .../edit-event-property.component.html | 1 - 70 files changed, 1690 insertions(+), 975 deletions(-) create mode 100644 streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/SupportsNestedTransformationRule.java rename streampipes-extensions/streampipes-connect-adapters/src/test/java/org/apache/streampipes/connect/adapters/generic/Mock.java => streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/convert/ProvidesConversionResult.java (69%) create mode 100644 streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/convert/SchemaConverter.java create mode 100644 streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/convert/ToOriginalSchemaConverter.java create mode 100644 streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/convert/ToTransformedSchemaConverter.java rename streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/elements/{AddValuePipelineElement.java => AdapterTransformationPipelineElement.java} (52%) delete mode 100644 streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/elements/AddTimestampPipelineElement.java delete mode 100644 streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/elements/TransformSchemaAdapterPipelineElement.java delete mode 100644 streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/elements/TransformStreamAdapterElement.java delete mode 100644 streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/elements/TransformValueAdapterPipelineElement.java create mode 100644 streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/generator/StatefulTransformationRuleGeneratorVisitor.java create mode 100644 streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/generator/StatelessTransformationRuleGeneratorVisitor.java rename streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/{transform/stream/StreamTransformationRule.java => generator/TransformationRuleGeneratorVisitor.java} (63%) rename streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/transform/schema/{SchemaTransformationRule.java => AddValueTransformationRule.java} (69%) delete mode 100644 streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/transform/schema/SchemaEventTransformer.java delete mode 100644 streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/transform/stream/StreamEventTransformer.java rename streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/transform/value/{ValueTransformationRule.java => AddTimestampTransformationRule.java} (71%) rename streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/transform/value/{TimestampTranformationRule.java => TimestampTransformationRule.java} (53%) delete mode 100644 streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/transform/value/ValueEventTransformer.java create mode 100644 streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/utils/ConversionUtils.java rename streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/{Util.java => utils/Utils.java} (71%) create mode 100644 streampipes-connect-shared/src/test/java/org/apache/streampipes/connect/shared/preprocessing/UtilsTest.java create mode 100644 streampipes-connect-shared/src/test/java/org/apache/streampipes/connect/shared/preprocessing/convert/Helpers.java create mode 100644 streampipes-connect-shared/src/test/java/org/apache/streampipes/connect/shared/preprocessing/convert/ToOriginalSchemaConverterTest.java create mode 100644 streampipes-connect-shared/src/test/java/org/apache/streampipes/connect/shared/preprocessing/convert/ToTransformedSchemaConverterTest.java create mode 100644 streampipes-connect-shared/src/test/java/org/apache/streampipes/connect/shared/preprocessing/elements/AdapterTransformationPipelineElementTest.java create mode 100644 streampipes-model/src/main/java/org/apache/streampipes/model/connect/rules/ITransformationRuleVisitor.java delete mode 100644 streampipes-model/src/main/java/org/apache/streampipes/model/connect/rules/value/TimestampTransformationRuleMode.java diff --git a/streampipes-connect-management/src/main/java/org/apache/streampipes/connect/management/AdapterEventPreviewPipeline.java b/streampipes-connect-management/src/main/java/org/apache/streampipes/connect/management/AdapterEventPreviewPipeline.java index 85c5ce427a..2e9b0fb241 100644 --- a/streampipes-connect-management/src/main/java/org/apache/streampipes/connect/management/AdapterEventPreviewPipeline.java +++ b/streampipes-connect-management/src/main/java/org/apache/streampipes/connect/management/AdapterEventPreviewPipeline.java @@ -41,7 +41,8 @@ public class AdapterEventPreviewPipeline implements IAdapterPipeline { public AdapterEventPreviewPipeline(AdapterEventPreview previewRequest) { this.objectMapper = new ObjectMapper(); - this.pipelineElements = new AdapterPipelineGeneratorBase().makeAdapterPipelineElements(previewRequest.getRules()); + this.pipelineElements = new AdapterPipelineGeneratorBase() + .makeAdapterPipelineElements(previewRequest.getRules(), false); this.event = previewRequest.getInputData(); } diff --git a/streampipes-connect-shared/pom.xml b/streampipes-connect-shared/pom.xml index 319b713fb5..82d056bbaf 100644 --- a/streampipes-connect-shared/pom.xml +++ b/streampipes-connect-shared/pom.xml @@ -46,11 +46,21 @@ 0.93.0-SNAPSHOT + + org.apache.streampipes + streampipes-sdk + 0.93.0-SNAPSHOT + junit junit test + + org.mockito + mockito-core + test + diff --git a/streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/AdapterPipelineGeneratorBase.java b/streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/AdapterPipelineGeneratorBase.java index ae20a96a20..727e2c431d 100644 --- a/streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/AdapterPipelineGeneratorBase.java +++ b/streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/AdapterPipelineGeneratorBase.java @@ -18,93 +18,30 @@ package org.apache.streampipes.connect.shared; -import org.apache.streampipes.connect.shared.preprocessing.elements.AddTimestampPipelineElement; -import org.apache.streampipes.connect.shared.preprocessing.elements.AddValuePipelineElement; -import org.apache.streampipes.connect.shared.preprocessing.elements.TransformSchemaAdapterPipelineElement; -import org.apache.streampipes.connect.shared.preprocessing.elements.TransformValueAdapterPipelineElement; +import org.apache.streampipes.connect.shared.preprocessing.elements.AdapterTransformationPipelineElement; +import org.apache.streampipes.connect.shared.preprocessing.generator.StatefulTransformationRuleGeneratorVisitor; +import org.apache.streampipes.connect.shared.preprocessing.generator.StatelessTransformationRuleGeneratorVisitor; import org.apache.streampipes.extensions.api.connect.IAdapterPipelineElement; import org.apache.streampipes.model.connect.rules.TransformationRuleDescription; -import org.apache.streampipes.model.connect.rules.schema.SchemaTransformationRuleDescription; -import org.apache.streampipes.model.connect.rules.stream.EventRateTransformationRuleDescription; -import org.apache.streampipes.model.connect.rules.stream.RemoveDuplicatesTransformationRuleDescription; -import org.apache.streampipes.model.connect.rules.value.AddTimestampRuleDescription; -import org.apache.streampipes.model.connect.rules.value.AddValueTransformationRuleDescription; -import org.apache.streampipes.model.connect.rules.value.ValueTransformationRuleDescription; import java.util.ArrayList; import java.util.List; -import java.util.stream.Collectors; public class AdapterPipelineGeneratorBase { - public List makeAdapterPipelineElements(List rules) { - List pipelineElements = new ArrayList<>(); - - // Must be before the schema transformations to ensure that user can move this event property - var timestampTransformationRuleDescription = getTimestampRule(rules); - if (timestampTransformationRuleDescription != null) { - pipelineElements.add(new AddTimestampPipelineElement( - timestampTransformationRuleDescription.getRuntimeKey())); - } - - var valueTransformationRuleDescription = getAddValueRule(rules); - if (valueTransformationRuleDescription != null) { - pipelineElements.add(new AddValuePipelineElement( - valueTransformationRuleDescription.getRuntimeKey(), - valueTransformationRuleDescription.getStaticValue())); - } - - // first transform schema before transforming vales - // value rules should use unique keys for of new schema - pipelineElements.add(new TransformSchemaAdapterPipelineElement(getSchemaRules(rules))); - pipelineElements.add(new TransformValueAdapterPipelineElement(getValueRules(rules))); - - return pipelineElements; - } - - protected RemoveDuplicatesTransformationRuleDescription getRemoveDuplicateRule( - List rules) { - return getRule(rules, RemoveDuplicatesTransformationRuleDescription.class); - } - - protected EventRateTransformationRuleDescription getEventRateTransformationRule( - List rules) { - return getRule(rules, EventRateTransformationRuleDescription.class); - } - - protected AddTimestampRuleDescription getTimestampRule(List rules) { - return getRule(rules, AddTimestampRuleDescription.class); - } - - protected AddValueTransformationRuleDescription getAddValueRule(List rules) { - return getRule(rules, AddValueTransformationRuleDescription.class); - } - - private T getRule(List rules, - Class type) { - - if (rules != null) { - for (TransformationRuleDescription tr : rules) { - if (type.isInstance(tr)) { - return type.cast(tr); - } - } + public List makeAdapterPipelineElements(List rules, + boolean includeStateful) { + var elements = new ArrayList(); + elements.add(new AdapterTransformationPipelineElement( + rules, + new StatelessTransformationRuleGeneratorVisitor()) + ); + if (includeStateful) { + elements.add(new AdapterTransformationPipelineElement( + rules, + new StatefulTransformationRuleGeneratorVisitor()) + ); } - - return null; - } - - private List getValueRules(List rules) { - return rules - .stream() - .filter(r -> r instanceof ValueTransformationRuleDescription && !(r instanceof AddTimestampRuleDescription)) - .collect(Collectors.toList()); - } - - private List getSchemaRules(List rules) { - return rules - .stream() - .filter(r -> r instanceof SchemaTransformationRuleDescription) - .collect(Collectors.toList()); + return elements; } } diff --git a/streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/SupportsNestedTransformationRule.java b/streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/SupportsNestedTransformationRule.java new file mode 100644 index 0000000000..d332a79e42 --- /dev/null +++ b/streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/SupportsNestedTransformationRule.java @@ -0,0 +1,56 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.streampipes.connect.shared.preprocessing; + +import org.apache.streampipes.connect.shared.preprocessing.transform.TransformationRule; + +import java.util.List; +import java.util.Map; + +public abstract class SupportsNestedTransformationRule implements TransformationRule { + + @Override + public Map apply(Map event) { + return applyNested(event, getEventKeys()); + } + + protected Map applyNested(Map event, + List eventKey) { + if (eventKey.size() == 1) { + applyTransformation(event, eventKey); + } else { + String key = eventKey.get(0); + List newKeysTmpList = eventKey.subList(1, eventKey.size()); + + Map newSubEvent = + applyNested((Map) event.get(eventKey.get(0)), newKeysTmpList); + + event.remove(key); + event.put(key, newSubEvent); + } + return event; + } + + protected abstract List getEventKeys(); + + protected abstract void applyTransformation(Map event, + List eventKey); + + +} diff --git a/streampipes-extensions/streampipes-connect-adapters/src/test/java/org/apache/streampipes/connect/adapters/generic/Mock.java b/streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/convert/ProvidesConversionResult.java similarity index 69% rename from streampipes-extensions/streampipes-connect-adapters/src/test/java/org/apache/streampipes/connect/adapters/generic/Mock.java rename to streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/convert/ProvidesConversionResult.java index e97c607177..bced5dac15 100644 --- a/streampipes-extensions/streampipes-connect-adapters/src/test/java/org/apache/streampipes/connect/adapters/generic/Mock.java +++ b/streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/convert/ProvidesConversionResult.java @@ -16,11 +16,14 @@ * */ -package org.apache.streampipes.connect.adapters.generic; +package org.apache.streampipes.connect.shared.preprocessing.convert; -public class Mock { - public static final int PORT = 8042; +import org.apache.streampipes.model.connect.rules.ITransformationRuleVisitor; +import org.apache.streampipes.model.schema.EventProperty; - public static final String HOST = "http://localhost:" + PORT; +import java.util.List; +public interface ProvidesConversionResult extends ITransformationRuleVisitor { + + List getTransformedProperties(); } diff --git a/streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/convert/SchemaConverter.java b/streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/convert/SchemaConverter.java new file mode 100644 index 0000000000..4f14239abe --- /dev/null +++ b/streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/convert/SchemaConverter.java @@ -0,0 +1,67 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.streampipes.connect.shared.preprocessing.convert; + +import org.apache.streampipes.model.connect.rules.TransformationRuleDescription; +import org.apache.streampipes.model.schema.EventSchema; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; +import java.util.List; +import java.util.stream.Collectors; + +public class SchemaConverter { + + public EventSchema toOriginalSchema(EventSchema transformedSchema, + List transformationRules) { + + var converter = new ToOriginalSchemaConverter( + transformedSchema.getEventProperties() + ); + + return transform(transformationRules, converter, true); + } + + public EventSchema toTransformedSchema(EventSchema originalSchema, + List transformationRules) { + + var converter = new ToTransformedSchemaConverter( + originalSchema.getEventProperties() + ); + + return transform(transformationRules, converter, false); + } + + private EventSchema transform(List transformationRules, + ProvidesConversionResult converter, + boolean reverseRules) { + var rules = transformationRules + .stream() + .sorted(Comparator.comparingInt(TransformationRuleDescription::getRulePriority)) + .collect(Collectors.toCollection(ArrayList::new)); + + if (reverseRules) { + Collections.reverse(rules); + } + + rules.forEach(rule -> rule.accept(converter)); + return new EventSchema(converter.getTransformedProperties()); + } +} diff --git a/streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/convert/ToOriginalSchemaConverter.java b/streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/convert/ToOriginalSchemaConverter.java new file mode 100644 index 0000000000..c35ed1a791 --- /dev/null +++ b/streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/convert/ToOriginalSchemaConverter.java @@ -0,0 +1,141 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.streampipes.connect.shared.preprocessing.convert; + +import org.apache.streampipes.connect.shared.preprocessing.utils.Utils; +import org.apache.streampipes.model.connect.rules.ITransformationRuleVisitor; +import org.apache.streampipes.model.connect.rules.schema.CreateNestedRuleDescription; +import org.apache.streampipes.model.connect.rules.schema.DeleteRuleDescription; +import org.apache.streampipes.model.connect.rules.schema.MoveRuleDescription; +import org.apache.streampipes.model.connect.rules.schema.RenameRuleDescription; +import org.apache.streampipes.model.connect.rules.stream.EventRateTransformationRuleDescription; +import org.apache.streampipes.model.connect.rules.stream.RemoveDuplicatesTransformationRuleDescription; +import org.apache.streampipes.model.connect.rules.value.AddTimestampRuleDescription; +import org.apache.streampipes.model.connect.rules.value.AddValueTransformationRuleDescription; +import org.apache.streampipes.model.connect.rules.value.ChangeDatatypeTransformationRuleDescription; +import org.apache.streampipes.model.connect.rules.value.CorrectionValueTransformationRuleDescription; +import org.apache.streampipes.model.connect.rules.value.TimestampTranfsformationRuleDescription; +import org.apache.streampipes.model.connect.rules.value.UnitTransformRuleDescription; +import org.apache.streampipes.model.schema.EventProperty; +import org.apache.streampipes.model.schema.EventPropertyNested; +import org.apache.streampipes.model.util.Cloner; +import org.apache.streampipes.sdk.helpers.EpProperties; +import org.apache.streampipes.sdk.helpers.Labels; +import org.apache.streampipes.sdk.utils.Datatypes; + +import java.net.URI; +import java.util.List; + +import static org.apache.streampipes.connect.shared.preprocessing.utils.ConversionUtils.findPrimitiveProperty; +import static org.apache.streampipes.connect.shared.preprocessing.utils.ConversionUtils.findProperty; +import static org.apache.streampipes.connect.shared.preprocessing.utils.ConversionUtils.findPropertyHierarchy; + + +public class ToOriginalSchemaConverter implements ITransformationRuleVisitor, ProvidesConversionResult { + + private final List properties; + + public ToOriginalSchemaConverter(List properties) { + this.properties = new Cloner().properties(properties); + } + + @Override + public void visit(CreateNestedRuleDescription rule) { + properties.removeIf(p -> p.getRuntimeName().equals(rule.getRuntimeKey())); + } + + @Override + public void visit(DeleteRuleDescription rule) { + // this can't be fully restored since we don't know the characteristics of the deleted field + // for now, we'll just add a string property with the deleted field name and enrich this after implementing #1960 + var propertyHierarchy = findPropertyHierarchy(properties, rule.getRuntimeKey()); + propertyHierarchy.add(EpProperties.stringEp(Labels.empty(), rule.getRuntimeKey(), "")); + } + + @Override + public void visit(MoveRuleDescription rule) { + var targetRuntimeKey = rule.getNewRuntimeKey() + "." + rule.getOldRuntimeKey(); + var existing = new Cloner().property(findProperty(properties, targetRuntimeKey)); + var existingHierarchy = findPropertyHierarchy(this.properties, targetRuntimeKey); + existingHierarchy.removeIf(property -> property.getRuntimeName().equals(existing.getRuntimeName())); + try { + var targetProperty = findProperty(this.properties, Utils.toKeyArray(rule.getOldRuntimeKey())); + if (targetProperty instanceof EventPropertyNested) { + ((EventPropertyNested) targetProperty).getEventProperties().add(existing); + } + } catch (IllegalArgumentException e) { + this.properties.add(existing); + } + } + + @Override + public void visit(RenameRuleDescription rule) { + var property = findProperty(properties, rule.getNewRuntimeKey()); + property.setRuntimeName(rule.getOldRuntimeKey()); + } + + @Override + public void visit(EventRateTransformationRuleDescription rule) { + // does not affect schema + } + + @Override + public void visit(RemoveDuplicatesTransformationRuleDescription rule) { + // does not affect schema + } + + @Override + public void visit(AddTimestampRuleDescription rule) { + properties.removeIf(p -> p.getRuntimeName().equals(rule.getRuntimeKey())); + } + + @Override + public void visit(AddValueTransformationRuleDescription rule) { + properties.removeIf(p -> p.getRuntimeName().equals(rule.getRuntimeKey())); + } + + @Override + public void visit(ChangeDatatypeTransformationRuleDescription rule) { + var property = findPrimitiveProperty(properties, rule.getRuntimeKey()); + property.setRuntimeType(rule.getOriginalDatatypeXsd()); + } + + @Override + public void visit(CorrectionValueTransformationRuleDescription rule) { + // does not affect schema + } + + @Override + public void visit(TimestampTranfsformationRuleDescription rule) { + var property = findPrimitiveProperty(properties, rule.getRuntimeKey()); + property.setRuntimeType(Datatypes.String.toString()); + property.setDomainProperties(List.of()); + } + + @Override + public void visit(UnitTransformRuleDescription rule) { + var property = findPrimitiveProperty(properties, rule.getRuntimeKey()); + property.setMeasurementUnit(URI.create(rule.getFromUnitRessourceURL())); + } + + @Override + public List getTransformedProperties() { + return properties; + } +} diff --git a/streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/convert/ToTransformedSchemaConverter.java b/streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/convert/ToTransformedSchemaConverter.java new file mode 100644 index 0000000000..bac84f5058 --- /dev/null +++ b/streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/convert/ToTransformedSchemaConverter.java @@ -0,0 +1,140 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.streampipes.connect.shared.preprocessing.convert; + +import org.apache.streampipes.connect.shared.preprocessing.utils.Utils; +import org.apache.streampipes.model.connect.rules.ITransformationRuleVisitor; +import org.apache.streampipes.model.connect.rules.schema.CreateNestedRuleDescription; +import org.apache.streampipes.model.connect.rules.schema.DeleteRuleDescription; +import org.apache.streampipes.model.connect.rules.schema.MoveRuleDescription; +import org.apache.streampipes.model.connect.rules.schema.RenameRuleDescription; +import org.apache.streampipes.model.connect.rules.stream.EventRateTransformationRuleDescription; +import org.apache.streampipes.model.connect.rules.stream.RemoveDuplicatesTransformationRuleDescription; +import org.apache.streampipes.model.connect.rules.value.AddTimestampRuleDescription; +import org.apache.streampipes.model.connect.rules.value.AddValueTransformationRuleDescription; +import org.apache.streampipes.model.connect.rules.value.ChangeDatatypeTransformationRuleDescription; +import org.apache.streampipes.model.connect.rules.value.CorrectionValueTransformationRuleDescription; +import org.apache.streampipes.model.connect.rules.value.TimestampTranfsformationRuleDescription; +import org.apache.streampipes.model.connect.rules.value.UnitTransformRuleDescription; +import org.apache.streampipes.model.schema.EventProperty; +import org.apache.streampipes.model.schema.EventPropertyNested; +import org.apache.streampipes.model.util.Cloner; +import org.apache.streampipes.sdk.helpers.EpProperties; +import org.apache.streampipes.sdk.helpers.Labels; +import org.apache.streampipes.sdk.utils.Datatypes; + +import java.net.URI; +import java.util.List; + +import static org.apache.streampipes.connect.shared.preprocessing.utils.ConversionUtils.findPrimitiveProperty; +import static org.apache.streampipes.connect.shared.preprocessing.utils.ConversionUtils.findProperty; +import static org.apache.streampipes.connect.shared.preprocessing.utils.ConversionUtils.findPropertyHierarchy; + +public class ToTransformedSchemaConverter implements ITransformationRuleVisitor, ProvidesConversionResult { + + private final List properties; + + public ToTransformedSchemaConverter(List properties) { + this.properties = new Cloner().properties(properties); + } + + @Override + public void visit(CreateNestedRuleDescription rule) { + var nested = new EventPropertyNested(); + nested.setRuntimeName(rule.getRuntimeKey()); + this.properties.add(nested); + } + + @Override + public void visit(DeleteRuleDescription rule) { + var keyArray = Utils.toKeyArray(rule.getRuntimeKey()); + var properties = findPropertyHierarchy(this.properties, rule.getRuntimeKey()); + properties.removeIf(property -> property.getRuntimeName().equals(keyArray.get(0))); + } + + @Override + public void visit(MoveRuleDescription rule) { + var existing = new Cloner().property(findProperty(properties, rule.getOldRuntimeKey())); + var existingHierarchy = findPropertyHierarchy(this.properties, rule.getOldRuntimeKey()); + existingHierarchy.removeIf(property -> property.getRuntimeName().equals(existing.getRuntimeName())); + try { + var targetProperty = findProperty(this.properties, Utils.toKeyArray(rule.getNewRuntimeKey())); + if (targetProperty instanceof EventPropertyNested) { + ((EventPropertyNested) targetProperty).getEventProperties().add(existing); + } + } catch (IllegalArgumentException e) { + this.properties.add(existing); + } + } + + @Override + public void visit(RenameRuleDescription rule) { + var property = findProperty(properties, rule.getOldRuntimeKey()); + property.setRuntimeName(rule.getNewRuntimeKey()); + } + + @Override + public void visit(EventRateTransformationRuleDescription rule) { + // does not affect schema + } + + @Override + public void visit(RemoveDuplicatesTransformationRuleDescription rule) { + // does not affect schema + } + + @Override + public void visit(AddTimestampRuleDescription rule) { + this.properties.add(EpProperties.timestampProperty(rule.getRuntimeKey())); + } + + @Override + public void visit(AddValueTransformationRuleDescription rule) { + this.properties.add(EpProperties.stringEp(Labels.empty(), rule.getRuntimeKey(), "")); + } + + @Override + public void visit(ChangeDatatypeTransformationRuleDescription rule) { + var property = findPrimitiveProperty(properties, rule.getRuntimeKey()); + property.setRuntimeType(rule.getTargetDatatypeXsd()); + } + + @Override + public void visit(CorrectionValueTransformationRuleDescription rule) { + // does not affect schema + } + + @Override + public void visit(TimestampTranfsformationRuleDescription rule) { + var property = findPrimitiveProperty(properties, rule.getRuntimeKey()); + property.setDomainProperties(List.of(URI.create("http://schema.org/DateTime"))); + property.setRuntimeType(Datatypes.Long.toString()); + } + + @Override + public void visit(UnitTransformRuleDescription rule) { + var property = findPrimitiveProperty(properties, rule.getRuntimeKey()); + property.setMeasurementUnit(URI.create(rule.getToUnitRessourceURL())); + } + + @Override + public List getTransformedProperties() { + return this.properties; + } +} diff --git a/streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/elements/AddValuePipelineElement.java b/streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/elements/AdapterTransformationPipelineElement.java similarity index 52% rename from streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/elements/AddValuePipelineElement.java rename to streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/elements/AdapterTransformationPipelineElement.java index ae61346254..d93f41f6ca 100644 --- a/streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/elements/AddValuePipelineElement.java +++ b/streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/elements/AdapterTransformationPipelineElement.java @@ -18,25 +18,32 @@ package org.apache.streampipes.connect.shared.preprocessing.elements; +import org.apache.streampipes.connect.shared.preprocessing.utils.Utils; +import org.apache.streampipes.connect.shared.preprocessing.generator.TransformationRuleGeneratorVisitor; +import org.apache.streampipes.connect.shared.preprocessing.transform.TransformationRule; import org.apache.streampipes.extensions.api.connect.IAdapterPipelineElement; +import org.apache.streampipes.model.connect.rules.TransformationRuleDescription; +import java.util.List; import java.util.Map; -public class AddValuePipelineElement implements IAdapterPipelineElement { +public class AdapterTransformationPipelineElement implements IAdapterPipelineElement { - private String runtimeKey; - private String value; + private final List transformationRules; + public AdapterTransformationPipelineElement(List transformationRules, + TransformationRuleGeneratorVisitor visitor) { + var descriptions = Utils.sortByPriority(transformationRules); - public AddValuePipelineElement(String runtimeKey, String value) { - this.runtimeKey = runtimeKey; - this.value = value; + descriptions.forEach(d -> d.accept(visitor)); + this.transformationRules = visitor.getTransformationRules(); } @Override public Map process(Map event) { - event.put(runtimeKey, value); + for (TransformationRule rule : transformationRules) { + event = rule.apply(event); + } return event; } - } diff --git a/streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/elements/AddTimestampPipelineElement.java b/streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/elements/AddTimestampPipelineElement.java deleted file mode 100644 index 22757a33f9..0000000000 --- a/streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/elements/AddTimestampPipelineElement.java +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package org.apache.streampipes.connect.shared.preprocessing.elements; - -import org.apache.streampipes.extensions.api.connect.IAdapterPipelineElement; - -import java.util.Map; - -public class AddTimestampPipelineElement implements IAdapterPipelineElement { - - private String runtimeKey; - - public AddTimestampPipelineElement(String runtimeKey) { - this.runtimeKey = runtimeKey; - } - - @Override - public Map process(Map event) { - event.put(runtimeKey, System.currentTimeMillis()); - return event; - } - -} diff --git a/streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/elements/TransformSchemaAdapterPipelineElement.java b/streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/elements/TransformSchemaAdapterPipelineElement.java deleted file mode 100644 index 5a6449750a..0000000000 --- a/streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/elements/TransformSchemaAdapterPipelineElement.java +++ /dev/null @@ -1,81 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package org.apache.streampipes.connect.shared.preprocessing.elements; - -import org.apache.streampipes.connect.shared.preprocessing.Util; -import org.apache.streampipes.connect.shared.preprocessing.transform.TransformationRule; -import org.apache.streampipes.connect.shared.preprocessing.transform.schema.CreateNestedTransformationRule; -import org.apache.streampipes.connect.shared.preprocessing.transform.schema.DeleteTransformationRule; -import org.apache.streampipes.connect.shared.preprocessing.transform.schema.MoveTransformationRule; -import org.apache.streampipes.connect.shared.preprocessing.transform.schema.RenameTransformationRule; -import org.apache.streampipes.connect.shared.preprocessing.transform.schema.SchemaEventTransformer; -import org.apache.streampipes.extensions.api.connect.IAdapterPipelineElement; -import org.apache.streampipes.model.connect.rules.TransformationRuleDescription; -import org.apache.streampipes.model.connect.rules.schema.CreateNestedRuleDescription; -import org.apache.streampipes.model.connect.rules.schema.DeleteRuleDescription; -import org.apache.streampipes.model.connect.rules.schema.MoveRuleDescription; -import org.apache.streampipes.model.connect.rules.schema.RenameRuleDescription; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.ArrayList; -import java.util.List; -import java.util.Map; - -public class TransformSchemaAdapterPipelineElement implements IAdapterPipelineElement { - - private SchemaEventTransformer eventTransformer; - Logger logger = LoggerFactory.getLogger(TransformSchemaAdapterPipelineElement.class); - - public TransformSchemaAdapterPipelineElement( - List transformationRuleDescriptions) { - List rules = new ArrayList<>(); - - // transforms description to actual rules - for (TransformationRuleDescription ruleDescription : transformationRuleDescriptions) { - if (ruleDescription instanceof RenameRuleDescription) { - RenameRuleDescription tmp = (RenameRuleDescription) ruleDescription; - rules.add(new RenameTransformationRule(Util.toKeyArray(tmp.getOldRuntimeKey()), - Util.getLastKey(tmp.getNewRuntimeKey()))); - } else if (ruleDescription instanceof MoveRuleDescription) { - MoveRuleDescription tmp = (MoveRuleDescription) ruleDescription; - rules.add(new MoveTransformationRule(Util.toKeyArray(tmp.getOldRuntimeKey()), - Util.toKeyArray(tmp.getNewRuntimeKey()))); - } else if (ruleDescription instanceof CreateNestedRuleDescription) { - CreateNestedRuleDescription tmp = (CreateNestedRuleDescription) ruleDescription; - rules.add(new CreateNestedTransformationRule(Util.toKeyArray(tmp.getRuntimeKey()))); - } else if (ruleDescription instanceof DeleteRuleDescription) { - DeleteRuleDescription tmp = (DeleteRuleDescription) ruleDescription; - rules.add(new DeleteTransformationRule(Util.toKeyArray(tmp.getRuntimeKey()))); - } else { - logger.error( - "Could not find the class for the rule description. This should never happen. " - + "Talk to admins to extend the rule implementations to get rid of this error!"); - } - } - - eventTransformer = new SchemaEventTransformer(rules); - } - - @Override - public Map process(Map event) { - return eventTransformer.transform(event); - } -} diff --git a/streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/elements/TransformStreamAdapterElement.java b/streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/elements/TransformStreamAdapterElement.java deleted file mode 100644 index 6504ac38da..0000000000 --- a/streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/elements/TransformStreamAdapterElement.java +++ /dev/null @@ -1,71 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package org.apache.streampipes.connect.shared.preprocessing.elements; - -import org.apache.streampipes.connect.shared.preprocessing.transform.TransformationRule; -import org.apache.streampipes.connect.shared.preprocessing.transform.stream.EventRateTransformationRule; -import org.apache.streampipes.connect.shared.preprocessing.transform.stream.StreamEventTransformer; -import org.apache.streampipes.extensions.api.connect.IAdapterPipelineElement; -import org.apache.streampipes.model.connect.rules.TransformationRuleDescription; -import org.apache.streampipes.model.connect.rules.stream.EventRateTransformationRuleDescription; -import org.apache.streampipes.model.connect.rules.stream.StreamTransformationRuleDescription; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.ArrayList; -import java.util.List; -import java.util.Map; - -public class TransformStreamAdapterElement implements IAdapterPipelineElement { - - private StreamEventTransformer eventTransformer; - Logger logger = LoggerFactory.getLogger(TransformStreamAdapterElement.class); - - public TransformStreamAdapterElement() { - eventTransformer = new StreamEventTransformer(); - } - - public TransformStreamAdapterElement(List transformationRuleDescriptions) { - List rules = new ArrayList<>(); - - // transforms description to actual rules - for (TransformationRuleDescription ruleDescription : transformationRuleDescriptions) { - if (ruleDescription instanceof EventRateTransformationRuleDescription) { - EventRateTransformationRuleDescription tmp = (EventRateTransformationRuleDescription) ruleDescription; - rules.add(new EventRateTransformationRule(tmp.getAggregationTimeWindow(), tmp.getAggregationType())); - } - } - - eventTransformer = new StreamEventTransformer(rules); - } - - public void addStreamTransformationRuleDescription(StreamTransformationRuleDescription ruleDescription) { - if (ruleDescription instanceof EventRateTransformationRuleDescription) { - EventRateTransformationRuleDescription tmp = (EventRateTransformationRuleDescription) ruleDescription; - eventTransformer.addEventRateTransformationRule( - new EventRateTransformationRule(tmp.getAggregationTimeWindow(), tmp.getAggregationType())); - } - } - - @Override - public Map process(Map event) { - return eventTransformer.transform(event); - } -} diff --git a/streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/elements/TransformValueAdapterPipelineElement.java b/streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/elements/TransformValueAdapterPipelineElement.java deleted file mode 100644 index 37aacbad31..0000000000 --- a/streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/elements/TransformValueAdapterPipelineElement.java +++ /dev/null @@ -1,96 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package org.apache.streampipes.connect.shared.preprocessing.elements; - -import org.apache.streampipes.connect.shared.preprocessing.Util; -import org.apache.streampipes.connect.shared.preprocessing.transform.value.CorrectionValueTransformationRule; -import org.apache.streampipes.connect.shared.preprocessing.transform.value.DatatypeTransformationRule; -import org.apache.streampipes.connect.shared.preprocessing.transform.value.TimestampTranformationRule; -import org.apache.streampipes.connect.shared.preprocessing.transform.value.TimestampTranformationRuleMode; -import org.apache.streampipes.connect.shared.preprocessing.transform.value.UnitTransformationRule; -import org.apache.streampipes.connect.shared.preprocessing.transform.value.ValueEventTransformer; -import org.apache.streampipes.connect.shared.preprocessing.transform.value.ValueTransformationRule; -import org.apache.streampipes.extensions.api.connect.IAdapterPipelineElement; -import org.apache.streampipes.model.connect.rules.TransformationRuleDescription; -import org.apache.streampipes.model.connect.rules.value.ChangeDatatypeTransformationRuleDescription; -import org.apache.streampipes.model.connect.rules.value.CorrectionValueTransformationRuleDescription; -import org.apache.streampipes.model.connect.rules.value.TimestampTranfsformationRuleDescription; -import org.apache.streampipes.model.connect.rules.value.UnitTransformRuleDescription; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.ArrayList; -import java.util.List; -import java.util.Map; - -public class TransformValueAdapterPipelineElement implements IAdapterPipelineElement { - - private final ValueEventTransformer eventTransformer; - private static final Logger logger = LoggerFactory.getLogger(TransformValueAdapterPipelineElement.class); - - public TransformValueAdapterPipelineElement( - List transformationRuleDescriptions) { - List rules = new ArrayList<>(); - - // transforms description to actual rules - for (TransformationRuleDescription ruleDescription : transformationRuleDescriptions) { - if (ruleDescription instanceof UnitTransformRuleDescription) { - var tmp = (UnitTransformRuleDescription) ruleDescription; - rules.add(new UnitTransformationRule(Util.toKeyArray(tmp.getRuntimeKey()), - tmp.getFromUnitRessourceURL(), tmp.getToUnitRessourceURL())); - } else if (ruleDescription instanceof TimestampTranfsformationRuleDescription) { - var tmp = (TimestampTranfsformationRuleDescription) ruleDescription; - TimestampTranformationRuleMode mode = null; - switch (tmp.getMode()) { - case "formatString": - mode = TimestampTranformationRuleMode.FORMAT_STRING; - break; - case "timeUnit": - mode = TimestampTranformationRuleMode.TIME_UNIT; - } - rules.add(new TimestampTranformationRule(Util.toKeyArray(tmp.getRuntimeKey()), mode, - tmp.getFormatString(), tmp.getMultiplier())); - } else if (ruleDescription instanceof CorrectionValueTransformationRuleDescription) { - var tmp = (CorrectionValueTransformationRuleDescription) ruleDescription; - rules.add(new CorrectionValueTransformationRule(Util.toKeyArray(tmp.getRuntimeKey()), tmp.getCorrectionValue(), - tmp.getOperator())); - } else if (ruleDescription instanceof ChangeDatatypeTransformationRuleDescription) { - var tmp = (ChangeDatatypeTransformationRuleDescription) ruleDescription; - rules.add(new DatatypeTransformationRule(tmp.getRuntimeKey(), tmp.getOriginalDatatypeXsd(), - tmp.getTargetDatatypeXsd())); - } else { - logger.error( - "Could not find the class for the rule description. This should never happen. " - + "Talk to admins to extend the rule implementations to get rid of this error!"); - } - } - - eventTransformer = new ValueEventTransformer(rules); - } - - @Override - public Map process(Map event) { - return eventTransformer.transform(event); - } - - public ValueEventTransformer getEventTransformer() { - return eventTransformer; - } -} diff --git a/streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/generator/StatefulTransformationRuleGeneratorVisitor.java b/streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/generator/StatefulTransformationRuleGeneratorVisitor.java new file mode 100644 index 0000000000..059cd4920d --- /dev/null +++ b/streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/generator/StatefulTransformationRuleGeneratorVisitor.java @@ -0,0 +1,99 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.streampipes.connect.shared.preprocessing.generator; + +import org.apache.streampipes.connect.shared.preprocessing.transform.stream.DuplicateFilterPipelineElement; +import org.apache.streampipes.connect.shared.preprocessing.transform.stream.EventRateTransformationRule; +import org.apache.streampipes.model.connect.rules.schema.CreateNestedRuleDescription; +import org.apache.streampipes.model.connect.rules.schema.DeleteRuleDescription; +import org.apache.streampipes.model.connect.rules.schema.MoveRuleDescription; +import org.apache.streampipes.model.connect.rules.schema.RenameRuleDescription; +import org.apache.streampipes.model.connect.rules.stream.EventRateTransformationRuleDescription; +import org.apache.streampipes.model.connect.rules.stream.RemoveDuplicatesTransformationRuleDescription; +import org.apache.streampipes.model.connect.rules.value.AddTimestampRuleDescription; +import org.apache.streampipes.model.connect.rules.value.AddValueTransformationRuleDescription; +import org.apache.streampipes.model.connect.rules.value.ChangeDatatypeTransformationRuleDescription; +import org.apache.streampipes.model.connect.rules.value.CorrectionValueTransformationRuleDescription; +import org.apache.streampipes.model.connect.rules.value.TimestampTranfsformationRuleDescription; +import org.apache.streampipes.model.connect.rules.value.UnitTransformRuleDescription; + +public class StatefulTransformationRuleGeneratorVisitor extends TransformationRuleGeneratorVisitor { + + @Override + public void visit(CreateNestedRuleDescription rule) { + + } + + @Override + public void visit(DeleteRuleDescription rule) { + + } + + @Override + public void visit(MoveRuleDescription rule) { + + } + + @Override + public void visit(RenameRuleDescription rule) { + + } + + @Override + public void visit(EventRateTransformationRuleDescription ruleDesc) { + rules.add( + new EventRateTransformationRule(ruleDesc.getAggregationTimeWindow(), ruleDesc.getAggregationType())); + } + + @Override + public void visit(RemoveDuplicatesTransformationRuleDescription ruleDesc) { + this.rules.add( + new DuplicateFilterPipelineElement(ruleDesc.getFilterTimeWindow())); + } + + @Override + public void visit(AddTimestampRuleDescription rule) { + + } + + @Override + public void visit(AddValueTransformationRuleDescription rule) { + + } + + @Override + public void visit(ChangeDatatypeTransformationRuleDescription rule) { + + } + + @Override + public void visit(CorrectionValueTransformationRuleDescription rule) { + + } + + @Override + public void visit(TimestampTranfsformationRuleDescription rule) { + + } + + @Override + public void visit(UnitTransformRuleDescription rule) { + + } +} diff --git a/streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/generator/StatelessTransformationRuleGeneratorVisitor.java b/streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/generator/StatelessTransformationRuleGeneratorVisitor.java new file mode 100644 index 0000000000..6692ff7aee --- /dev/null +++ b/streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/generator/StatelessTransformationRuleGeneratorVisitor.java @@ -0,0 +1,134 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.streampipes.connect.shared.preprocessing.generator; + +import org.apache.streampipes.connect.shared.preprocessing.utils.Utils; +import org.apache.streampipes.connect.shared.preprocessing.transform.schema.AddValueTransformationRule; +import org.apache.streampipes.connect.shared.preprocessing.transform.schema.CreateNestedTransformationRule; +import org.apache.streampipes.connect.shared.preprocessing.transform.schema.DeleteTransformationRule; +import org.apache.streampipes.connect.shared.preprocessing.transform.schema.MoveTransformationRule; +import org.apache.streampipes.connect.shared.preprocessing.transform.schema.RenameTransformationRule; +import org.apache.streampipes.connect.shared.preprocessing.transform.value.AddTimestampTransformationRule; +import org.apache.streampipes.connect.shared.preprocessing.transform.value.CorrectionValueTransformationRule; +import org.apache.streampipes.connect.shared.preprocessing.transform.value.DatatypeTransformationRule; +import org.apache.streampipes.connect.shared.preprocessing.transform.value.TimestampTranformationRuleMode; +import org.apache.streampipes.connect.shared.preprocessing.transform.value.TimestampTransformationRule; +import org.apache.streampipes.connect.shared.preprocessing.transform.value.UnitTransformationRule; +import org.apache.streampipes.model.connect.rules.schema.CreateNestedRuleDescription; +import org.apache.streampipes.model.connect.rules.schema.DeleteRuleDescription; +import org.apache.streampipes.model.connect.rules.schema.MoveRuleDescription; +import org.apache.streampipes.model.connect.rules.schema.RenameRuleDescription; +import org.apache.streampipes.model.connect.rules.stream.EventRateTransformationRuleDescription; +import org.apache.streampipes.model.connect.rules.stream.RemoveDuplicatesTransformationRuleDescription; +import org.apache.streampipes.model.connect.rules.value.AddTimestampRuleDescription; +import org.apache.streampipes.model.connect.rules.value.AddValueTransformationRuleDescription; +import org.apache.streampipes.model.connect.rules.value.ChangeDatatypeTransformationRuleDescription; +import org.apache.streampipes.model.connect.rules.value.CorrectionValueTransformationRuleDescription; +import org.apache.streampipes.model.connect.rules.value.TimestampTranfsformationRuleDescription; +import org.apache.streampipes.model.connect.rules.value.UnitTransformRuleDescription; + +public class StatelessTransformationRuleGeneratorVisitor extends TransformationRuleGeneratorVisitor { + + @Override + public void visit(CreateNestedRuleDescription ruleDesc) { + rules.add(new CreateNestedTransformationRule( + Utils.toKeyArray(ruleDesc.getRuntimeKey()))); + } + + @Override + public void visit(DeleteRuleDescription ruleDesc) { + rules.add(new DeleteTransformationRule( + Utils.toKeyArray(ruleDesc.getRuntimeKey()))); + } + + @Override + public void visit(MoveRuleDescription ruleDesc) { + rules.add(new MoveTransformationRule( + Utils.toKeyArray(ruleDesc.getOldRuntimeKey()), + Utils.toKeyArray(ruleDesc.getNewRuntimeKey()))); + } + + @Override + public void visit(RenameRuleDescription ruleDesc) { + rules.add(new RenameTransformationRule( + Utils.toKeyArray(ruleDesc.getOldRuntimeKey()), + Utils.getLastKey(ruleDesc.getNewRuntimeKey()))); + } + + @Override + public void visit(EventRateTransformationRuleDescription ruleDesc) { + // Do nothing + } + + @Override + public void visit(RemoveDuplicatesTransformationRuleDescription ruleDesc) { + // Do nothing + } + + @Override + public void visit(AddTimestampRuleDescription ruleDesc) { + rules.add(new AddTimestampTransformationRule( + ruleDesc.getRuntimeKey())); + } + + @Override + public void visit(AddValueTransformationRuleDescription ruleDesc) { + rules.add(new AddValueTransformationRule( + ruleDesc.getRuntimeKey(), + ruleDesc.getStaticValue())); + } + + @Override + public void visit(ChangeDatatypeTransformationRuleDescription ruleDesc) { + rules.add(new DatatypeTransformationRule( + ruleDesc.getRuntimeKey(), + ruleDesc.getOriginalDatatypeXsd(), + ruleDesc.getTargetDatatypeXsd())); + } + + @Override + public void visit(CorrectionValueTransformationRuleDescription ruleDesc) { + rules.add(new CorrectionValueTransformationRule( + Utils.toKeyArray(ruleDesc.getRuntimeKey()), + ruleDesc.getCorrectionValue(), + ruleDesc.getOperator())); + } + + @Override + public void visit(TimestampTranfsformationRuleDescription ruleDesc) { + TimestampTranformationRuleMode mode; + if (ruleDesc.getMode().equals(TimestampTranformationRuleMode.FORMAT_STRING.internalName())) { + mode = TimestampTranformationRuleMode.FORMAT_STRING; + } else { + mode = TimestampTranformationRuleMode.FORMAT_STRING; + } + + rules.add(new TimestampTransformationRule(Utils.toKeyArray( + ruleDesc.getRuntimeKey()), + mode, + ruleDesc.getFormatString(), + ruleDesc.getMultiplier())); + } + + @Override + public void visit(UnitTransformRuleDescription ruleDesc) { + rules.add(new UnitTransformationRule(ruleDesc)); + } + +} diff --git a/streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/transform/stream/StreamTransformationRule.java b/streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/generator/TransformationRuleGeneratorVisitor.java similarity index 63% rename from streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/transform/stream/StreamTransformationRule.java rename to streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/generator/TransformationRuleGeneratorVisitor.java index b264eceda0..2b6a4edaff 100644 --- a/streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/transform/stream/StreamTransformationRule.java +++ b/streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/generator/TransformationRuleGeneratorVisitor.java @@ -16,9 +16,23 @@ * */ -package org.apache.streampipes.connect.shared.preprocessing.transform.stream; +package org.apache.streampipes.connect.shared.preprocessing.generator; import org.apache.streampipes.connect.shared.preprocessing.transform.TransformationRule; +import org.apache.streampipes.model.connect.rules.ITransformationRuleVisitor; -public interface StreamTransformationRule extends TransformationRule { +import java.util.ArrayList; +import java.util.List; + +public abstract class TransformationRuleGeneratorVisitor implements ITransformationRuleVisitor { + + protected final List rules; + + public TransformationRuleGeneratorVisitor() { + this.rules = new ArrayList<>(); + } + + public List getTransformationRules() { + return rules; + } } diff --git a/streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/transform/TransformationRule.java b/streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/transform/TransformationRule.java index e1cffa2f32..f6443afc62 100644 --- a/streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/transform/TransformationRule.java +++ b/streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/transform/TransformationRule.java @@ -21,5 +21,5 @@ import java.util.Map; public interface TransformationRule { - Map transform(Map event); + Map apply(Map event); } diff --git a/streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/transform/schema/SchemaTransformationRule.java b/streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/transform/schema/AddValueTransformationRule.java similarity index 69% rename from streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/transform/schema/SchemaTransformationRule.java rename to streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/transform/schema/AddValueTransformationRule.java index 47b8281395..c1b8ff688a 100644 --- a/streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/transform/schema/SchemaTransformationRule.java +++ b/streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/transform/schema/AddValueTransformationRule.java @@ -20,5 +20,22 @@ import org.apache.streampipes.connect.shared.preprocessing.transform.TransformationRule; -public interface SchemaTransformationRule extends TransformationRule { +import java.util.Map; + +public class AddValueTransformationRule implements TransformationRule { + + private final String runtimeKey; + private final String value; + + public AddValueTransformationRule(String runtimeKey, String value) { + this.runtimeKey = runtimeKey; + this.value = value; + } + + @Override + public Map apply(Map event) { + event.put(runtimeKey, value); + return event; + } + } diff --git a/streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/transform/schema/CreateNestedTransformationRule.java b/streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/transform/schema/CreateNestedTransformationRule.java index 411ebd6806..5d2e158b63 100644 --- a/streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/transform/schema/CreateNestedTransformationRule.java +++ b/streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/transform/schema/CreateNestedTransformationRule.java @@ -18,38 +18,27 @@ package org.apache.streampipes.connect.shared.preprocessing.transform.schema; +import org.apache.streampipes.connect.shared.preprocessing.SupportsNestedTransformationRule; + import java.util.HashMap; import java.util.List; import java.util.Map; -public class CreateNestedTransformationRule implements SchemaTransformationRule { - private List key; +public class CreateNestedTransformationRule extends SupportsNestedTransformationRule { + + private final List key; public CreateNestedTransformationRule(List key) { this.key = key; } @Override - public Map transform(Map event) { - return transform(event, key); + protected List getEventKeys() { + return key; } - private Map transform(Map event, List keys) { - - if (keys.size() == 1) { - event.put(keys.get(0), new HashMap<>()); - return event; - } else { - String key = keys.get(0); - List newKeysTmpList = keys.subList(1, keys.size()); - - Map newSubEvent = - transform((Map) event.get(keys.get(0)), newKeysTmpList); - - event.remove(key); - event.put(key, newSubEvent); - return event; - } - + @Override + protected void applyTransformation(Map event, List eventKeys) { + event.put(eventKeys.get(0), new HashMap<>()); } } diff --git a/streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/transform/schema/DeleteTransformationRule.java b/streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/transform/schema/DeleteTransformationRule.java index bda4047c03..11fb3abe4c 100644 --- a/streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/transform/schema/DeleteTransformationRule.java +++ b/streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/transform/schema/DeleteTransformationRule.java @@ -18,37 +18,26 @@ package org.apache.streampipes.connect.shared.preprocessing.transform.schema; +import org.apache.streampipes.connect.shared.preprocessing.SupportsNestedTransformationRule; + import java.util.List; import java.util.Map; -public class DeleteTransformationRule implements SchemaTransformationRule { +public class DeleteTransformationRule extends SupportsNestedTransformationRule { - private List key; + private final List key; public DeleteTransformationRule(List key) { this.key = key; } @Override - public Map transform(Map event) { - return transform(event, key); + protected List getEventKeys() { + return key; } - private Map transform(Map event, List keys) { - if (keys.size() == 1) { - event.remove(keys.get(0)); - return event; - } else { - String key = keys.get(0); - List newKeysTmpList = keys.subList(1, keys.size()); - - Map newSubEvent = - transform((Map) event.get(keys.get(0)), newKeysTmpList); - - event.remove(key); - event.put(key, newSubEvent); - return event; - } - + @Override + protected void applyTransformation(Map event, List eventKeys) { + event.remove(eventKeys.get(0)); } } diff --git a/streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/transform/schema/MoveTransformationRule.java b/streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/transform/schema/MoveTransformationRule.java index 598ce038ef..6bb5bdcd33 100644 --- a/streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/transform/schema/MoveTransformationRule.java +++ b/streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/transform/schema/MoveTransformationRule.java @@ -18,28 +18,24 @@ package org.apache.streampipes.connect.shared.preprocessing.transform.schema; +import org.apache.streampipes.connect.shared.preprocessing.transform.TransformationRule; + import java.util.HashMap; import java.util.List; import java.util.Map; -public class MoveTransformationRule implements SchemaTransformationRule { +public class MoveTransformationRule implements TransformationRule { - private List oldKey; - private List newKey; + private final List oldKey; + private final List newKey; public MoveTransformationRule(List oldKey, List newKey) { this.oldKey = oldKey; - -// List tmp = new ArrayList<>(); -// for (int i = 0; i < newKey.size() - 1; i++) { -// tmp.add(newKey.get(i)); -// } - this.newKey = newKey; } @Override - public Map transform(Map event) { + public Map apply(Map event) { Map objectToMove = (Map) ((HashMap) getItem(event, oldKey)).clone(); @@ -51,7 +47,7 @@ public Map transform(Map event) { } private Map addItem(Map event, List keys, Map movedObject) { - if (keys.size() == 0 || (keys.size() == 1 && keys.get(0).equals(""))) { + if (keys.isEmpty() || (keys.size() == 1 && keys.get(0).isEmpty())) { String key = (String) movedObject.keySet().toArray()[0]; event.put(key, movedObject.get(key)); return event; diff --git a/streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/transform/schema/RenameTransformationRule.java b/streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/transform/schema/RenameTransformationRule.java index d41e7d46ae..c710d2418b 100644 --- a/streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/transform/schema/RenameTransformationRule.java +++ b/streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/transform/schema/RenameTransformationRule.java @@ -18,12 +18,14 @@ package org.apache.streampipes.connect.shared.preprocessing.transform.schema; +import org.apache.streampipes.connect.shared.preprocessing.SupportsNestedTransformationRule; + import java.util.List; import java.util.Map; -public class RenameTransformationRule implements SchemaTransformationRule { - private List oldKey; - private String newKey; +public class RenameTransformationRule extends SupportsNestedTransformationRule { + private final List oldKey; + private final String newKey; public RenameTransformationRule(List oldKey, String newKey) { this.oldKey = oldKey; @@ -31,29 +33,15 @@ public RenameTransformationRule(List oldKey, String newKey) { } @Override - public Map transform(Map event) { - Map nestedEvent = event; - - return transform(event, oldKey); + protected List getEventKeys() { + return oldKey; } - private Map transform(Map event, List keys) { - if (keys.size() == 1) { - Object o = event.get(keys.get(0)); - event.remove(keys.get(0)); - event.put(newKey, o); - - } else { - String key = keys.get(0); - List newKeysTmpList = keys.subList(1, keys.size()); - Map newSubEvent = - transform((Map) event.get(key), newKeysTmpList); - - event.remove(key); - event.put(key, newSubEvent); - } - - return event; + @Override + protected void applyTransformation(Map event, List eventKeys) { + Object o = event.get(eventKeys.get(0)); + event.remove(eventKeys.get(0)); + event.put(newKey, o); } } diff --git a/streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/transform/schema/SchemaEventTransformer.java b/streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/transform/schema/SchemaEventTransformer.java deleted file mode 100644 index bfd68681c8..0000000000 --- a/streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/transform/schema/SchemaEventTransformer.java +++ /dev/null @@ -1,119 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package org.apache.streampipes.connect.shared.preprocessing.transform.schema; - -import org.apache.streampipes.connect.shared.preprocessing.transform.TransformationRule; - -import java.util.ArrayList; -import java.util.List; -import java.util.Map; - -public class SchemaEventTransformer implements SchemaTransformationRule { - - private List renameTransformationRules; - private List createNestedTransformationRules; - private List moveTransformationRules; - private List deleteTransformationRules; - - public SchemaEventTransformer(List rules) { - this.renameTransformationRules = new ArrayList<>(); - this.createNestedTransformationRules = new ArrayList<>(); - this.moveTransformationRules = new ArrayList<>(); - this.deleteTransformationRules = new ArrayList<>(); - - for (TransformationRule rule : rules) { - if (rule instanceof RenameTransformationRule) { - this.renameTransformationRules.add((RenameTransformationRule) rule); - } else if (rule instanceof CreateNestedTransformationRule) { - this.createNestedTransformationRules.add((CreateNestedTransformationRule) rule); - } else if (rule instanceof MoveTransformationRule) { - this.moveTransformationRules.add((MoveTransformationRule) rule); - } else if (rule instanceof DeleteTransformationRule) { - this.deleteTransformationRules.add((DeleteTransformationRule) rule); - } - } - } - - - public SchemaEventTransformer(List renameTransformationRules, - List createNestedTransformationRules, - List moveTransformationRules, - List deleteTransformationRules) { - this.renameTransformationRules = renameTransformationRules; - this.createNestedTransformationRules = createNestedTransformationRules; - this.moveTransformationRules = moveTransformationRules; - this.deleteTransformationRules = deleteTransformationRules; - } - - - @Override - public Map transform(Map event) { - - for (RenameTransformationRule renameRule : renameTransformationRules) { - event = renameRule.transform(event); - } - - for (CreateNestedTransformationRule createRule : createNestedTransformationRules) { - event = createRule.transform(event); - } - - for (MoveTransformationRule moveRule : moveTransformationRules) { - event = moveRule.transform(event); - } - - for (DeleteTransformationRule deleteRule : deleteTransformationRules) { - event = deleteRule.transform(event); - } - - return event; - } - - - public List getRenameTransformationRules() { - return renameTransformationRules; - } - - public void setRenameTransformationRules(List renameTransformationRules) { - this.renameTransformationRules = renameTransformationRules; - } - - public List getCreateNestedTransformationRules() { - return createNestedTransformationRules; - } - - public void setCreateNestedTransformationRules(List createNestedTransformationRules) { - this.createNestedTransformationRules = createNestedTransformationRules; - } - - public List getMoveTransformationRules() { - return moveTransformationRules; - } - - public void setMoveTransformationRules(List moveTransformationRules) { - this.moveTransformationRules = moveTransformationRules; - } - - public List getDeleteTransformationRules() { - return deleteTransformationRules; - } - - public void setDeleteTransformationRules(List deleteTransformationRules) { - this.deleteTransformationRules = deleteTransformationRules; - } -} diff --git a/streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/transform/stream/DuplicateFilterPipelineElement.java b/streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/transform/stream/DuplicateFilterPipelineElement.java index 55da491baa..bbca98fb1e 100644 --- a/streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/transform/stream/DuplicateFilterPipelineElement.java +++ b/streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/transform/stream/DuplicateFilterPipelineElement.java @@ -18,7 +18,7 @@ package org.apache.streampipes.connect.shared.preprocessing.transform.stream; -import org.apache.streampipes.extensions.api.connect.IAdapterPipelineElement; +import org.apache.streampipes.connect.shared.preprocessing.transform.TransformationRule; import java.util.HashMap; import java.util.Map; @@ -28,12 +28,12 @@ * If the same event is sent multiple times the timer is always reseted to cover polling of rest endpoints * User can configure how long events are stored in cache, it should be minimum 2x the polling intervall */ -public class DuplicateFilterPipelineElement implements IAdapterPipelineElement { +public class DuplicateFilterPipelineElement implements TransformationRule { /** * Lifetime of events */ - private long filterTimeWindow; + private final long filterTimeWindow; public DuplicateFilterPipelineElement(String filterTimeWindow) { // convert it to seconds @@ -47,7 +47,7 @@ public DuplicateFilterPipelineElement(String filterTimeWindow) { private long lastCleanUpTimestamp = System.currentTimeMillis(); @Override - public Map process(Map event) { + public Map apply(Map event) { cleanUpEvenState(); int hash = event.hashCode(); diff --git a/streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/transform/stream/EventRateTransformationRule.java b/streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/transform/stream/EventRateTransformationRule.java index 916f633f24..229952c8b3 100644 --- a/streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/transform/stream/EventRateTransformationRule.java +++ b/streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/transform/stream/EventRateTransformationRule.java @@ -18,14 +18,16 @@ package org.apache.streampipes.connect.shared.preprocessing.transform.stream; +import org.apache.streampipes.connect.shared.preprocessing.transform.TransformationRule; + import java.util.Map; -public class EventRateTransformationRule implements StreamTransformationRule { +public class EventRateTransformationRule implements TransformationRule { - private long aggregationTimeWindow; + private final long aggregationTimeWindow; //none (Values from last event), max, min, mean, sum (of the values in the time window) - private String aggregationType; + private final String aggregationType; private long lastSentToPipelineTimestamp = System.currentTimeMillis(); @@ -35,7 +37,7 @@ public EventRateTransformationRule(long aggregationTimeWindow, String aggregatio } @Override - public Map transform(Map event) { + public Map apply(Map event) { if (System.currentTimeMillis() > lastSentToPipelineTimestamp + aggregationTimeWindow) { switch (aggregationType) { case "none": diff --git a/streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/transform/stream/StreamEventTransformer.java b/streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/transform/stream/StreamEventTransformer.java deleted file mode 100644 index a9beb550ee..0000000000 --- a/streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/transform/stream/StreamEventTransformer.java +++ /dev/null @@ -1,66 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package org.apache.streampipes.connect.shared.preprocessing.transform.stream; - -import org.apache.streampipes.connect.shared.preprocessing.transform.TransformationRule; - -import java.util.ArrayList; -import java.util.List; -import java.util.Map; - -public class StreamEventTransformer implements StreamTransformationRule { - - private List eventRateTransformationRules; - - public StreamEventTransformer() { - this.eventRateTransformationRules = new ArrayList<>(); - } - - - public StreamEventTransformer(List rules) { - this(); - - for (TransformationRule rule : rules) { - if (rule instanceof EventRateTransformationRule) { - this.eventRateTransformationRules.add((EventRateTransformationRule) rule); - } - } - } - - - /* - public StreamEventTransformer(List eventRateTransformationRules) { - this.eventRateTransformationRules = eventRateTransformationRules; - } - */ - - public void addEventRateTransformationRule(EventRateTransformationRule rule) { - this.eventRateTransformationRules.add(rule); - } - - @Override - public Map transform(Map event) { - - for (EventRateTransformationRule rateRule : eventRateTransformationRules) { - event = rateRule.transform(event); - } - - return event; - } -} diff --git a/streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/transform/value/ValueTransformationRule.java b/streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/transform/value/AddTimestampTransformationRule.java similarity index 71% rename from streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/transform/value/ValueTransformationRule.java rename to streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/transform/value/AddTimestampTransformationRule.java index 9d767959ee..c8267ef34a 100644 --- a/streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/transform/value/ValueTransformationRule.java +++ b/streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/transform/value/AddTimestampTransformationRule.java @@ -20,5 +20,19 @@ import org.apache.streampipes.connect.shared.preprocessing.transform.TransformationRule; -public interface ValueTransformationRule extends TransformationRule { +import java.util.Map; + +public class AddTimestampTransformationRule implements TransformationRule { + + private final String runtimeKey; + + public AddTimestampTransformationRule(String runtimeKey) { + this.runtimeKey = runtimeKey; + } + + @Override + public Map apply(Map event) { + event.put(runtimeKey, System.currentTimeMillis()); + return event; + } } diff --git a/streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/transform/value/CorrectionValueTransformationRule.java b/streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/transform/value/CorrectionValueTransformationRule.java index 44835c8274..2101743ae2 100644 --- a/streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/transform/value/CorrectionValueTransformationRule.java +++ b/streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/transform/value/CorrectionValueTransformationRule.java @@ -18,13 +18,15 @@ package org.apache.streampipes.connect.shared.preprocessing.transform.value; +import org.apache.streampipes.connect.shared.preprocessing.SupportsNestedTransformationRule; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.util.List; import java.util.Map; -public class CorrectionValueTransformationRule implements ValueTransformationRule { +public class CorrectionValueTransformationRule extends SupportsNestedTransformationRule { private static Logger logger = LoggerFactory.getLogger(CorrectionValueTransformationRule.class); @@ -40,50 +42,34 @@ public CorrectionValueTransformationRule(List keys, } @Override - public Map transform(Map event) { - return transform(event, eventKey); + protected List getEventKeys() { + return eventKey; } - private Map transform(Map event, List eventKey) { - - if (eventKey.size() == 1) { - try { - Object obj = event.get(eventKey.get(0)); - double old; - if (obj instanceof Number) { - old = ((Number) obj).doubleValue(); - } else { - throw new RuntimeException( - String.format("Selected property `%s` does not contain a numeric value: `%s", eventKey.get(0), obj) - ); - } - - double corrected = switch (operator) { - case "MULTIPLY" -> old * correctionValue; - case "ADD" -> old + correctionValue; - case "SUBTRACT" -> old - correctionValue; - case "DIVIDE" -> old / correctionValue; - default -> old; - }; - - event.put(eventKey.get(0), corrected); - } catch (ClassCastException e) { - logger.error(e.toString()); + @Override + protected void applyTransformation(Map event, List eventKey) { + try { + Object obj = event.get(eventKey.get(0)); + double old; + if (obj instanceof Number) { + old = ((Number) obj).doubleValue(); + } else { + throw new RuntimeException( + String.format("Selected property `%s` does not contain a numeric value: `%s", eventKey.get(0), obj) + ); } - return event; - } else { - String key = eventKey.get(0); - List newKeysTmpList = eventKey.subList(1, eventKey.size()); - - Map newSubEvent = - transform((Map) event.get(eventKey.get(0)), newKeysTmpList); - - event.remove(key); - event.put(key, newSubEvent); - - return event; + double corrected = switch (operator) { + case "MULTIPLY" -> old * correctionValue; + case "ADD" -> old + correctionValue; + case "SUBTRACT" -> old - correctionValue; + case "DIVIDE" -> old / correctionValue; + default -> old; + }; + + event.put(eventKey.get(0), corrected); + } catch (ClassCastException e) { + logger.error(e.toString()); } - } } diff --git a/streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/transform/value/DatatypeTransformationRule.java b/streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/transform/value/DatatypeTransformationRule.java index d24b809921..ce0f0509a5 100644 --- a/streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/transform/value/DatatypeTransformationRule.java +++ b/streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/transform/value/DatatypeTransformationRule.java @@ -20,28 +20,24 @@ import org.apache.streampipes.connect.shared.DatatypeUtils; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +import org.apache.streampipes.connect.shared.preprocessing.transform.TransformationRule; import java.util.Map; -public class DatatypeTransformationRule implements ValueTransformationRule { - - private static final Logger LOG = LoggerFactory.getLogger(DatatypeTransformationRule.class); +public class DatatypeTransformationRule implements TransformationRule { - private String eventKey; - private String originalDatatypeXsd; + private final String eventKey; private String targetDatatypeXsd; - public DatatypeTransformationRule(String eventKey, String originalDatatypeXsd, String targetDatatypeXsd) { + public DatatypeTransformationRule(String eventKey, + String originalDatatypeXsd, + String targetDatatypeXsd) { this.eventKey = eventKey; - this.originalDatatypeXsd = originalDatatypeXsd; this.targetDatatypeXsd = targetDatatypeXsd; } @Override - public Map transform(Map event) { + public Map apply(Map event) { Object value = event.get(eventKey); Object transformedValue = transformDatatype(value); event.put(eventKey, transformedValue); diff --git a/streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/transform/value/TimestampTranformationRuleMode.java b/streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/transform/value/TimestampTranformationRuleMode.java index b579e0b53e..32431e5aca 100644 --- a/streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/transform/value/TimestampTranformationRuleMode.java +++ b/streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/transform/value/TimestampTranformationRuleMode.java @@ -20,7 +20,16 @@ public enum TimestampTranformationRuleMode { - TIME_UNIT, - FORMAT_STRING, + TIME_UNIT("timeUnit"), + FORMAT_STRING("formatString"); + private final String internalName; + + TimestampTranformationRuleMode(String internalName) { + this.internalName = internalName; + } + + public String internalName() { + return internalName; + } } diff --git a/streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/transform/value/TimestampTranformationRule.java b/streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/transform/value/TimestampTransformationRule.java similarity index 53% rename from streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/transform/value/TimestampTranformationRule.java rename to streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/transform/value/TimestampTransformationRule.java index aa5f3a9fdd..5f9f30de61 100644 --- a/streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/transform/value/TimestampTranformationRule.java +++ b/streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/transform/value/TimestampTransformationRule.java @@ -18,6 +18,8 @@ package org.apache.streampipes.connect.shared.preprocessing.transform.value; +import org.apache.streampipes.connect.shared.preprocessing.SupportsNestedTransformationRule; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -26,22 +28,22 @@ import java.util.List; import java.util.Map; -public class TimestampTranformationRule implements ValueTransformationRule { +public class TimestampTransformationRule extends SupportsNestedTransformationRule { - private List eventKey; - private TimestampTranformationRuleMode mode; - private String formatString; - private long multiplier; + private final List eventKey; + private final TimestampTranformationRuleMode mode; + private final long multiplier; private SimpleDateFormat dateFormatter; - private static Logger logger = LoggerFactory.getLogger(TimestampTranformationRule.class); + private static final Logger logger = LoggerFactory.getLogger(TimestampTransformationRule.class); - public TimestampTranformationRule(List eventKey, TimestampTranformationRuleMode mode, String formatString, - long multiplier) { + public TimestampTransformationRule(List eventKey, + TimestampTranformationRuleMode mode, + String formatString, + long multiplier) { this.eventKey = eventKey; this.mode = mode; - this.formatString = formatString; this.multiplier = multiplier; if (mode == TimestampTranformationRuleMode.FORMAT_STRING) { @@ -50,37 +52,21 @@ public TimestampTranformationRule(List eventKey, TimestampTranformationR } @Override - public Map transform(Map event) { - return transform(event, eventKey); + protected List getEventKeys() { + return eventKey; } - private Map transform(Map event, List eventKey) { - - if (eventKey.size() == 1) { - - switch (mode) { - case TIME_UNIT: - long timeLong = Long.valueOf(String.valueOf(event.get(eventKey.get(0)))); - event.put(eventKey.get(0), this.performTimeUnitTransformation(timeLong)); - break; - case FORMAT_STRING: - String dateString = String.valueOf(event.get(eventKey.get(0))); - event.put(eventKey.get(0), performFormatStringTransformation(dateString)); + @Override + protected void applyTransformation(Map event, List eventKey) { + switch (mode) { + case TIME_UNIT -> { + long timeLong = Long.parseLong(String.valueOf(event.get(eventKey.get(0)))); + event.put(eventKey.get(0), this.performTimeUnitTransformation(timeLong)); + } + case FORMAT_STRING -> { + String dateString = String.valueOf(event.get(eventKey.get(0))); + event.put(eventKey.get(0), performFormatStringTransformation(dateString)); } - - return event; - - } else { - String key = eventKey.get(0); - List newKeysTmpList = eventKey.subList(1, eventKey.size()); - - Map newSubEvent = - transform((Map) event.get(eventKey.get(0)), newKeysTmpList); - - event.remove(key); - event.put(key, newSubEvent); - - return event; } } diff --git a/streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/transform/value/UnitTransformationRule.java b/streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/transform/value/UnitTransformationRule.java index e353fd2a4a..1dabbd6fac 100644 --- a/streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/transform/value/UnitTransformationRule.java +++ b/streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/transform/value/UnitTransformationRule.java @@ -18,6 +18,9 @@ package org.apache.streampipes.connect.shared.preprocessing.transform.value; +import org.apache.streampipes.connect.shared.preprocessing.SupportsNestedTransformationRule; +import org.apache.streampipes.connect.shared.preprocessing.utils.Utils; +import org.apache.streampipes.model.connect.rules.value.UnitTransformRuleDescription; import org.apache.streampipes.units.UnitProvider; import com.github.jqudt.Quantity; @@ -28,13 +31,19 @@ import java.util.List; import java.util.Map; -public class UnitTransformationRule implements ValueTransformationRule { +public class UnitTransformationRule extends SupportsNestedTransformationRule { - private static Logger logger = LoggerFactory.getLogger(UnitTransformationRule.class); + private static final Logger logger = LoggerFactory.getLogger(UnitTransformationRule.class); - private List eventKey; - private Unit unitTypeFrom; - private Unit unitTypeTo; + private final List eventKey; + private final Unit unitTypeFrom; + private final Unit unitTypeTo; + + public UnitTransformationRule(UnitTransformRuleDescription description) { + this.unitTypeFrom = UnitProvider.INSTANCE.getUnit(description.getFromUnitRessourceURL()); + this.unitTypeTo = UnitProvider.INSTANCE.getUnit(description.getToUnitRessourceURL()); + this.eventKey = Utils.toKeyArray(description.getRuntimeKey()); + } public UnitTransformationRule(List keys, String fromUnitRessourceURL, String toUnitRessourceURL) { @@ -44,39 +53,21 @@ public UnitTransformationRule(List keys, } @Override - public Map transform(Map event) { - return transform(event, eventKey); + protected List getEventKeys() { + return eventKey; } - private Map transform(Map event, List eventKey) { - - if (eventKey.size() == 1) { - try { - double value = Double.valueOf(String.valueOf(event.get(eventKey.get(0)))); - - Quantity obs = new Quantity(value, unitTypeFrom); - double newValue = obs.convertTo(unitTypeTo).getValue(); - - event.put(eventKey.get(0), newValue); - } catch (ClassCastException e) { - logger.error(e.toString()); - } catch (IllegalAccessException e) { - logger.error(e.toString()); - } - return event; - - } else { - String key = eventKey.get(0); - List newKeysTmpList = eventKey.subList(1, eventKey.size()); - - Map newSubEvent = - transform((Map) event.get(eventKey.get(0)), newKeysTmpList); + @Override + protected void applyTransformation(Map event, List eventKey) { + try { + double value = Double.parseDouble(String.valueOf(event.get(eventKey.get(0)))); - event.remove(key); - event.put(key, newSubEvent); + Quantity obs = new Quantity(value, unitTypeFrom); + double newValue = obs.convertTo(unitTypeTo).getValue(); - return event; + event.put(eventKey.get(0), newValue); + } catch (ClassCastException | IllegalAccessException e) { + logger.error(e.toString()); } - } } diff --git a/streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/transform/value/ValueEventTransformer.java b/streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/transform/value/ValueEventTransformer.java deleted file mode 100644 index 963eb99a52..0000000000 --- a/streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/transform/value/ValueEventTransformer.java +++ /dev/null @@ -1,83 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package org.apache.streampipes.connect.shared.preprocessing.transform.value; - -import org.apache.streampipes.connect.shared.preprocessing.transform.TransformationRule; - -import java.util.ArrayList; -import java.util.List; -import java.util.Map; - -public class ValueEventTransformer implements ValueTransformationRule { - - private final List unitTransformationRules; - private List timestampTransformationRules; - private final List correctionValueTransformationRules; - private final List datatypeTransformationRules; - - public ValueEventTransformer(List rules) { - this.unitTransformationRules = new ArrayList<>(); - this.timestampTransformationRules = new ArrayList<>(); - this.correctionValueTransformationRules = new ArrayList<>(); - this.datatypeTransformationRules = new ArrayList<>(); - - for (TransformationRule rule : rules) { - if (rule instanceof UnitTransformationRule) { - this.unitTransformationRules.add((UnitTransformationRule) rule); - } else if (rule instanceof TimestampTranformationRule) { - this.timestampTransformationRules.add((TimestampTranformationRule) rule); - } else if (rule instanceof CorrectionValueTransformationRule) { - this.correctionValueTransformationRules.add((CorrectionValueTransformationRule) rule); - } else if (rule instanceof DatatypeTransformationRule) { - this.datatypeTransformationRules.add((DatatypeTransformationRule) rule); - } - } - } - - @Override - public Map transform(Map event) { - - for (UnitTransformationRule rule : unitTransformationRules) { - event = rule.transform(event); - } - - for (TimestampTranformationRule rule : timestampTransformationRules) { - event = rule.transform(event); - } - - for (var rule : datatypeTransformationRules) { - event = rule.transform(event); - } - - for (CorrectionValueTransformationRule rule : correctionValueTransformationRules) { - event = rule.transform(event); - } - - return event; - } - - public List getTimestampTransformationRules() { - return timestampTransformationRules; - } - - public void setTimestampTransformationRules( - List timestampTransformationRules) { - this.timestampTransformationRules = timestampTransformationRules; - } -} diff --git a/streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/utils/ConversionUtils.java b/streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/utils/ConversionUtils.java new file mode 100644 index 0000000000..646f58128c --- /dev/null +++ b/streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/utils/ConversionUtils.java @@ -0,0 +1,93 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.streampipes.connect.shared.preprocessing.utils; + +import org.apache.streampipes.model.schema.EventProperty; +import org.apache.streampipes.model.schema.EventPropertyList; +import org.apache.streampipes.model.schema.EventPropertyNested; +import org.apache.streampipes.model.schema.EventPropertyPrimitive; + +import java.util.Collections; +import java.util.List; + +public class ConversionUtils { + + public static EventProperty findProperty(List properties, + String runtimeKey) { + return findProperty(properties, Utils.toKeyArray(runtimeKey)); + } + + public static EventPropertyPrimitive findPrimitiveProperty(List properties, + String runtimeKey) { + return (EventPropertyPrimitive) findProperty(properties, runtimeKey); + } + + public static List findPropertyHierarchy(List originalProperties, + String runtimeKeys) { + return findPropertyHierarchy(originalProperties, Utils.toKeyArray(runtimeKeys)); + } + + public static List findPropertyHierarchy(List originalProperties, + List runtimeKeys) { + if (runtimeKeys.isEmpty()) { + return Collections.emptyList(); + } else if (runtimeKeys.size() == 1) { + return originalProperties; + } else { + List finalRuntimeKeys = runtimeKeys; + var properties = originalProperties + .stream() + .filter(ep -> ep.getRuntimeName().equals(finalRuntimeKeys.get(0))) + .toList(); + if (!properties.isEmpty()) { + runtimeKeys = runtimeKeys.subList(1, runtimeKeys.size()); + EventPropertyNested nestedProperty = (EventPropertyNested) properties.get(0); + return findPropertyHierarchy(nestedProperty.getEventProperties(), runtimeKeys); + } else { + return Collections.emptyList(); + } + } + } + + public static EventProperty findProperty(List originalProperties, + List runtimeKeys) { + if (runtimeKeys.isEmpty()) { + throw new IllegalArgumentException("Could not find property"); + } + + String firstRuntimeKey = runtimeKeys.get(0); + + for (EventProperty property : originalProperties) { + if (property.getRuntimeName().equals(firstRuntimeKey)) { + if (property instanceof EventPropertyPrimitive || property instanceof EventPropertyList) { + return property; + } else if (runtimeKeys.size() > 1) { + return findProperty( + ((EventPropertyNested) property).getEventProperties(), + runtimeKeys.subList(1, runtimeKeys.size()) + ); + } else { + return property; + } + } + } + + throw new IllegalArgumentException("Could not find property"); + } +} diff --git a/streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/Util.java b/streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/utils/Utils.java similarity index 71% rename from streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/Util.java rename to streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/utils/Utils.java index 0087b03204..50a5a5681c 100644 --- a/streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/Util.java +++ b/streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/utils/Utils.java @@ -16,12 +16,15 @@ * */ -package org.apache.streampipes.connect.shared.preprocessing; +package org.apache.streampipes.connect.shared.preprocessing.utils; + +import org.apache.streampipes.model.connect.rules.TransformationRuleDescription; import java.util.Arrays; +import java.util.Comparator; import java.util.List; -public class Util { +public class Utils { public static String getLastKey(String s) { String[] list = s.split("\\."); @@ -35,10 +38,17 @@ public static String getLastKey(String s) { public static List toKeyArray(String s) { String[] split = s.split("\\."); if (split.length == 0) { - return Arrays.asList(s); + return List.of(s); } else { return Arrays.asList(split); } } + public static List sortByPriority(List rules) { + return rules + .stream() + .sorted(Comparator.comparingInt(TransformationRuleDescription::getRulePriority)) + .toList(); + } + } diff --git a/streampipes-connect-shared/src/test/java/org/apache/streampipes/connect/shared/preprocessing/UtilsTest.java b/streampipes-connect-shared/src/test/java/org/apache/streampipes/connect/shared/preprocessing/UtilsTest.java new file mode 100644 index 0000000000..c372aed60f --- /dev/null +++ b/streampipes-connect-shared/src/test/java/org/apache/streampipes/connect/shared/preprocessing/UtilsTest.java @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.streampipes.connect.shared.preprocessing; + +import org.apache.streampipes.connect.shared.preprocessing.utils.Utils; +import org.apache.streampipes.model.connect.rules.schema.RenameRuleDescription; +import org.apache.streampipes.model.connect.rules.value.AddTimestampRuleDescription; + +import org.junit.Test; + +import java.util.List; + +import static org.junit.Assert.assertEquals; + +public class UtilsTest { + + @Test + public void testSortByPriority() { + var renameTransformationRule = new RenameRuleDescription(); + var addTimestampTransformationRule = new AddTimestampRuleDescription(); + + var original = List.of(renameTransformationRule, addTimestampTransformationRule); + var sorted = Utils.sortByPriority(original); + + assertEquals(2, sorted.size()); + assertEquals(sorted.get(0).getRulePriority(), 100); + assertEquals(sorted.get(1).getRulePriority(), 210); + } +} diff --git a/streampipes-connect-shared/src/test/java/org/apache/streampipes/connect/shared/preprocessing/convert/Helpers.java b/streampipes-connect-shared/src/test/java/org/apache/streampipes/connect/shared/preprocessing/convert/Helpers.java new file mode 100644 index 0000000000..ca8dd0c347 --- /dev/null +++ b/streampipes-connect-shared/src/test/java/org/apache/streampipes/connect/shared/preprocessing/convert/Helpers.java @@ -0,0 +1,92 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.streampipes.connect.shared.preprocessing.convert; + +import org.apache.streampipes.model.connect.rules.schema.DeleteRuleDescription; +import org.apache.streampipes.model.connect.rules.schema.MoveRuleDescription; +import org.apache.streampipes.model.connect.rules.schema.RenameRuleDescription; +import org.apache.streampipes.model.connect.rules.value.UnitTransformRuleDescription; +import org.apache.streampipes.model.schema.EventProperty; +import org.apache.streampipes.model.schema.EventPropertyNested; +import org.apache.streampipes.model.schema.EventPropertyPrimitive; +import org.apache.streampipes.sdk.helpers.EpProperties; +import org.apache.streampipes.sdk.helpers.Labels; + +import java.util.ArrayList; +import java.util.List; + +public class Helpers { + + public static String getUnit(EventProperty eventProperty) { + return ((EventPropertyPrimitive) eventProperty).getMeasurementUnit().toString(); + } + + public static List makeSimpleProperties(boolean addTimestamp) { + List properties = new ArrayList<>(); + properties.add(EpProperties.stringEp(Labels.empty(), "stringProp", "")); + properties.add(EpProperties.integerEp(Labels.empty(), "intProp", "")); + if (addTimestamp) { + properties.add(EpProperties.timestampProperty("timestamp")); + } + return properties; + } + + public static List makeNestedProperties() { + List properties = new ArrayList<>(); + properties.add(EpProperties.timestampProperty("timestamp")); + var nestedProperties = makeSimpleProperties(false); + var nestedProperty = new EventPropertyNested(); + nestedProperty.setRuntimeName("nested"); + nestedProperty.setEventProperties(nestedProperties); + properties.add(nestedProperty); + return properties; + } + + public static UnitTransformRuleDescription makeUnitTransformationRule(String runtimeKey) { + var rule = new UnitTransformRuleDescription(); + rule.setRuntimeKey(runtimeKey); + rule.setFromUnitRessourceURL("originalUnit"); + rule.setToUnitRessourceURL("targetUnit"); + + return rule; + } + + public static MoveRuleDescription makeMoveTransformationRule(String runtimeKey, + String newRuntimeKey) { + var rule = new MoveRuleDescription(); + rule.setOldRuntimeKey(runtimeKey); + rule.setNewRuntimeKey(newRuntimeKey); + return rule; + } + + public static RenameRuleDescription makeRenameTransformationRule(String runtimeKey, + String newRuntimeName) { + var rule = new RenameRuleDescription(); + rule.setOldRuntimeKey(runtimeKey); + rule.setNewRuntimeKey(newRuntimeName); + + return rule; + } + + public static DeleteRuleDescription makeDeleteTransformationRule(String runtimeKey) { + var rule = new DeleteRuleDescription(); + rule.setRuntimeKey(runtimeKey); + return rule; + } +} diff --git a/streampipes-connect-shared/src/test/java/org/apache/streampipes/connect/shared/preprocessing/convert/ToOriginalSchemaConverterTest.java b/streampipes-connect-shared/src/test/java/org/apache/streampipes/connect/shared/preprocessing/convert/ToOriginalSchemaConverterTest.java new file mode 100644 index 0000000000..b42986d8c4 --- /dev/null +++ b/streampipes-connect-shared/src/test/java/org/apache/streampipes/connect/shared/preprocessing/convert/ToOriginalSchemaConverterTest.java @@ -0,0 +1,104 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.streampipes.connect.shared.preprocessing.convert; + +import org.apache.streampipes.model.connect.rules.TransformationRuleDescription; +import org.apache.streampipes.model.schema.EventProperty; +import org.apache.streampipes.model.schema.EventPropertyNested; +import org.apache.streampipes.model.schema.EventSchema; +import org.apache.streampipes.sdk.helpers.EpProperties; +import org.apache.streampipes.sdk.helpers.Labels; + +import org.junit.Test; + +import java.util.ArrayList; +import java.util.List; + +import static org.apache.streampipes.connect.shared.preprocessing.convert.Helpers.getUnit; +import static org.apache.streampipes.connect.shared.preprocessing.convert.Helpers.makeDeleteTransformationRule; +import static org.apache.streampipes.connect.shared.preprocessing.convert.Helpers.makeMoveTransformationRule; +import static org.apache.streampipes.connect.shared.preprocessing.convert.Helpers.makeNestedProperties; +import static org.apache.streampipes.connect.shared.preprocessing.convert.Helpers.makeSimpleProperties; +import static org.apache.streampipes.connect.shared.preprocessing.convert.Helpers.makeUnitTransformationRule; +import static org.junit.Assert.assertEquals; + +public class ToOriginalSchemaConverterTest { + + @Test + public void testSimpleUnitConversion() { + List properties = makeSimpleProperties(true); + + var rules = new ArrayList(); + + rules.add(makeUnitTransformationRule("stringProp")); + + var resultProperties = executeAndReturnResult(properties, rules); + + assertEquals(3, resultProperties.size()); + assertEquals("originalUnit", getUnit(resultProperties.get(0))); + } + + @Test + public void testNestedUnitConversion() { + List properties = makeNestedProperties(); + + var rules = new ArrayList(); + + rules.add(makeUnitTransformationRule("nested.stringProp")); + + var resultProperties = executeAndReturnResult(properties, rules); + var nestedResultProperty = ((EventPropertyNested) resultProperties.get(1)).getEventProperties().get(0); + + assertEquals(2, resultProperties.size()); + assertEquals("originalUnit", getUnit(nestedResultProperty)); + } + + @Test + public void testSimpleMoveConversion() { + List properties = makeNestedProperties(); + var nestedProperty = ((EventPropertyNested) properties.get(1)); + nestedProperty.getEventProperties().add(EpProperties.stringEp(Labels.empty(), "epToBeMoved", "")); + + var rules = new ArrayList(); + + rules.add(makeMoveTransformationRule("epToBeMoved", "nested")); + + var result = executeAndReturnResult(properties, rules); + assertEquals(3, result.size()); + assertEquals("timestamp", result.get(0).getRuntimeName()); + assertEquals(2, ((EventPropertyNested) result.get(1)).getEventProperties().size()); + } + + @Test + public void testDeleteRule() { + List properties = makeSimpleProperties(true); + var rules = new ArrayList(); + + rules.add(makeDeleteTransformationRule("epToBeRestored")); + var result = executeAndReturnResult(properties, rules); + assertEquals(4, result.size()); + assertEquals("epToBeRestored", result.get(3).getRuntimeName()); + } + + private List executeAndReturnResult(List properties, + List rules) { + var result = new SchemaConverter().toOriginalSchema(new EventSchema(properties), rules); + return result.getEventProperties(); + } +} diff --git a/streampipes-connect-shared/src/test/java/org/apache/streampipes/connect/shared/preprocessing/convert/ToTransformedSchemaConverterTest.java b/streampipes-connect-shared/src/test/java/org/apache/streampipes/connect/shared/preprocessing/convert/ToTransformedSchemaConverterTest.java new file mode 100644 index 0000000000..1eb6855fd4 --- /dev/null +++ b/streampipes-connect-shared/src/test/java/org/apache/streampipes/connect/shared/preprocessing/convert/ToTransformedSchemaConverterTest.java @@ -0,0 +1,121 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.streampipes.connect.shared.preprocessing.convert; + +import org.apache.streampipes.model.connect.rules.TransformationRuleDescription; +import org.apache.streampipes.model.schema.EventProperty; +import org.apache.streampipes.model.schema.EventPropertyNested; +import org.apache.streampipes.model.schema.EventSchema; +import org.apache.streampipes.sdk.helpers.EpProperties; +import org.apache.streampipes.sdk.helpers.Labels; + +import org.junit.Test; + +import java.util.ArrayList; +import java.util.List; + +import static org.apache.streampipes.connect.shared.preprocessing.convert.Helpers.getUnit; +import static org.apache.streampipes.connect.shared.preprocessing.convert.Helpers.makeDeleteTransformationRule; +import static org.apache.streampipes.connect.shared.preprocessing.convert.Helpers.makeMoveTransformationRule; +import static org.apache.streampipes.connect.shared.preprocessing.convert.Helpers.makeNestedProperties; +import static org.apache.streampipes.connect.shared.preprocessing.convert.Helpers.makeRenameTransformationRule; +import static org.apache.streampipes.connect.shared.preprocessing.convert.Helpers.makeSimpleProperties; +import static org.apache.streampipes.connect.shared.preprocessing.convert.Helpers.makeUnitTransformationRule; +import static org.junit.Assert.assertEquals; + +public class ToTransformedSchemaConverterTest { + + @Test + public void testSimpleUnitConversion() { + List properties = makeSimpleProperties(true); + + var rules = new ArrayList(); + + rules.add(makeUnitTransformationRule("stringProp")); + + var resultProperties = executeAndReturnResult(properties, rules); + + assertEquals(3, resultProperties.size()); + assertEquals("targetUnit", getUnit(resultProperties.get(0))); + } + + + @Test + public void testSimpleRenameConversion() { + List properties = makeSimpleProperties(true); + var rules = new ArrayList(); + + rules.add(makeRenameTransformationRule("stringProp", "newStringProp")); + + var result = executeAndReturnResult(properties, rules); + assertEquals(3, result.size()); + assertEquals("newStringProp", result.get(0).getRuntimeName()); + } + + @Test + public void testSimpleDeleteConversion() { + List properties = makeSimpleProperties(true); + var rules = new ArrayList(); + + rules.add(makeDeleteTransformationRule("stringProp")); + + var result = executeAndReturnResult(properties, rules); + assertEquals(2, result.size()); + assertEquals("intProp", result.get(0).getRuntimeName()); + } + + @Test + public void testSimpleMoveConversion() { + List properties = makeNestedProperties(); + properties.add(EpProperties.stringEp(Labels.empty(), "epToBeMoved", "")); + + var rules = new ArrayList(); + + rules.add(makeMoveTransformationRule("epToBeMoved", "nested")); + + var result = executeAndReturnResult(properties, rules); + assertEquals(2, result.size()); + assertEquals("timestamp", result.get(0).getRuntimeName()); + assertEquals(3, ((EventPropertyNested) result.get(1)).getEventProperties().size()); + } + + @Test + public void testNestedUnitConversion() { + List properties = makeNestedProperties(); + + var rules = new ArrayList(); + + rules.add(makeUnitTransformationRule("nested.stringProp")); + + var resultProperties = executeAndReturnResult(properties, rules); + var nestedResultProperty = ((EventPropertyNested) resultProperties.get(1)).getEventProperties().get(0); + + assertEquals(2, resultProperties.size()); + assertEquals("targetUnit", getUnit(nestedResultProperty)); + } + + private List executeAndReturnResult(List properties, + List rules) { + var result = new SchemaConverter().toTransformedSchema(new EventSchema(properties), rules); + return result.getEventProperties(); + } + + + +} diff --git a/streampipes-connect-shared/src/test/java/org/apache/streampipes/connect/shared/preprocessing/elements/AdapterTransformationPipelineElementTest.java b/streampipes-connect-shared/src/test/java/org/apache/streampipes/connect/shared/preprocessing/elements/AdapterTransformationPipelineElementTest.java new file mode 100644 index 0000000000..6dd6b8d497 --- /dev/null +++ b/streampipes-connect-shared/src/test/java/org/apache/streampipes/connect/shared/preprocessing/elements/AdapterTransformationPipelineElementTest.java @@ -0,0 +1,60 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.streampipes.connect.shared.preprocessing.elements; + +import org.apache.streampipes.connect.shared.preprocessing.generator.StatelessTransformationRuleGeneratorVisitor; +import org.apache.streampipes.model.connect.rules.schema.RenameRuleDescription; +import org.apache.streampipes.model.connect.rules.value.AddTimestampRuleDescription; + +import org.junit.Test; +import org.mockito.Mockito; + +import java.util.HashMap; +import java.util.List; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.mockito.Mockito.when; + +public class AdapterTransformationPipelineElementTest { + + @Test + public void testTransformationPipelineElement() { + var renameRuleDescription = new RenameRuleDescription(); + var spy = Mockito.spy(renameRuleDescription); + when(spy.getOldRuntimeKey()).thenReturn("temperature"); + when(spy.getNewRuntimeKey()).thenReturn("temp"); + + var addTimestampRule = new AddTimestampRuleDescription("timestamp"); + + var rules = List.of(spy, addTimestampRule); + + var pipelineElement = new AdapterTransformationPipelineElement(rules, new StatelessTransformationRuleGeneratorVisitor()); + var event = new HashMap(); + event.put("temperature", 1); + + var result = pipelineElement.process(event); + + assertEquals(2, result.size()); + assertTrue(result.containsKey("temp")); + assertFalse(result.containsKey("temperature")); + assertTrue(result.containsKey("timestamp")); + } +} diff --git a/streampipes-connect-shared/src/test/java/org/apache/streampipes/connect/shared/preprocessing/transform/schema/CreateNestedTransformationRuleTest.java b/streampipes-connect-shared/src/test/java/org/apache/streampipes/connect/shared/preprocessing/transform/schema/CreateNestedTransformationRuleTest.java index 9b15b043f0..a48a9be2c2 100644 --- a/streampipes-connect-shared/src/test/java/org/apache/streampipes/connect/shared/preprocessing/transform/schema/CreateNestedTransformationRuleTest.java +++ b/streampipes-connect-shared/src/test/java/org/apache/streampipes/connect/shared/preprocessing/transform/schema/CreateNestedTransformationRuleTest.java @@ -39,7 +39,7 @@ public void transformSimple() { key.add("key"); CreateNestedTransformationRule createNested = new CreateNestedTransformationRule(key); - Map result = createNested.transform(event); + Map result = createNested.apply(event); assertEquals(1, result.keySet().size()); assertEquals(0, ((Map) result.get("key")).keySet().size()); @@ -58,7 +58,7 @@ public void transformNested() { CreateNestedTransformationRule createNested = new CreateNestedTransformationRule(key); - Map result = createNested.transform(event); + Map result = createNested.apply(event); assertEquals(1, result.keySet().size()); assertEquals(1, ((Map) result.get("parent")).keySet().size()); diff --git a/streampipes-connect-shared/src/test/java/org/apache/streampipes/connect/shared/preprocessing/transform/schema/DeleteTransformationRuleTest.java b/streampipes-connect-shared/src/test/java/org/apache/streampipes/connect/shared/preprocessing/transform/schema/DeleteTransformationRuleTest.java index a4549b5841..38a99ebf16 100644 --- a/streampipes-connect-shared/src/test/java/org/apache/streampipes/connect/shared/preprocessing/transform/schema/DeleteTransformationRuleTest.java +++ b/streampipes-connect-shared/src/test/java/org/apache/streampipes/connect/shared/preprocessing/transform/schema/DeleteTransformationRuleTest.java @@ -36,7 +36,7 @@ public void transformSimple() { DeleteTransformationRule deleteRule = new DeleteTransformationRule(List.of("key")); - Map result = deleteRule.transform(event); + Map result = deleteRule.apply(event); assertEquals(0, result.keySet().size()); } @@ -50,7 +50,7 @@ public void transformNested() { DeleteTransformationRule deleteRule = new DeleteTransformationRule(Arrays.asList("parent", "child")); - Map result = deleteRule.transform(event); + Map result = deleteRule.apply(event); assertEquals(1, result.keySet().size()); } diff --git a/streampipes-connect-shared/src/test/java/org/apache/streampipes/connect/shared/preprocessing/transform/schema/MoveTransformationRuleTest.java b/streampipes-connect-shared/src/test/java/org/apache/streampipes/connect/shared/preprocessing/transform/schema/MoveTransformationRuleTest.java index a46dd0ea51..77ba8ec4cb 100644 --- a/streampipes-connect-shared/src/test/java/org/apache/streampipes/connect/shared/preprocessing/transform/schema/MoveTransformationRuleTest.java +++ b/streampipes-connect-shared/src/test/java/org/apache/streampipes/connect/shared/preprocessing/transform/schema/MoveTransformationRuleTest.java @@ -51,7 +51,7 @@ public void transform() { MoveTransformationRule moveRule = new MoveTransformationRule(oldKey, newKey); - Map result = moveRule.transform(event); + Map result = moveRule.apply(event); assertEquals(2, result.keySet().size()); assertEquals(0, ((Map) result.get("old_parent")).keySet().size()); @@ -66,7 +66,7 @@ public void transFormTopLevelProperty() { MoveTransformationRule moveRule = new MoveTransformationRule(Arrays.asList("toMove"), Arrays.asList("new_parent")); - Map result = moveRule.transform(event); + Map result = moveRule.apply(event); assertEquals(1, result.keySet().size()); assertEquals(1, ((Map) result.get("new_parent")).keySet().size()); @@ -82,7 +82,7 @@ public void transToTopLevelProperty() { MoveTransformationRule moveRule = new MoveTransformationRule(Arrays.asList("parent", "child"), Arrays.asList("")); - Map result = moveRule.transform(parent); + Map result = moveRule.apply(parent); assertEquals(2, result.keySet().size()); assertEquals(0, ((Map) result.get("parent")).keySet().size()); diff --git a/streampipes-connect-shared/src/test/java/org/apache/streampipes/connect/shared/preprocessing/transform/schema/RenameTransformationRuleTest.java b/streampipes-connect-shared/src/test/java/org/apache/streampipes/connect/shared/preprocessing/transform/schema/RenameTransformationRuleTest.java index 555bb68e7a..d773dd63cd 100644 --- a/streampipes-connect-shared/src/test/java/org/apache/streampipes/connect/shared/preprocessing/transform/schema/RenameTransformationRuleTest.java +++ b/streampipes-connect-shared/src/test/java/org/apache/streampipes/connect/shared/preprocessing/transform/schema/RenameTransformationRuleTest.java @@ -38,7 +38,7 @@ public void renameSimple() { oldKey.add("old_key"); RenameTransformationRule renameRule = new RenameTransformationRule(oldKey, "new_key"); - Map result = renameRule.transform(event); + Map result = renameRule.apply(event); assertEquals(1, result.keySet().size()); assertEquals("test", result.get("new_key")); @@ -58,7 +58,7 @@ public void renameNested() { oldKey.add("old_key"); RenameTransformationRule renameRule = new RenameTransformationRule(oldKey, "new_key"); - Map result = renameRule.transform(event); + Map result = renameRule.apply(event); assertEquals(1, result.keySet().size()); Map resultNested = (Map) result.get("key"); diff --git a/streampipes-connect-shared/src/test/java/org/apache/streampipes/connect/shared/preprocessing/transform/schema/SchemaEventTransformerTest.java b/streampipes-connect-shared/src/test/java/org/apache/streampipes/connect/shared/preprocessing/transform/schema/SchemaEventTransformerTest.java index e572ebdb18..a4e630e5fd 100644 --- a/streampipes-connect-shared/src/test/java/org/apache/streampipes/connect/shared/preprocessing/transform/schema/SchemaEventTransformerTest.java +++ b/streampipes-connect-shared/src/test/java/org/apache/streampipes/connect/shared/preprocessing/transform/schema/SchemaEventTransformerTest.java @@ -35,7 +35,7 @@ public class SchemaEventTransformerTest { @Test @SuppressWarnings("unchecked") public void transform() { - Map event = getFirstEvent(); + Map result = getFirstEvent(); List rules = new ArrayList<>(); rules.add(new RenameTransformationRule(List.of("a"), "a1")); @@ -46,10 +46,9 @@ public void transform() { rules.add(new MoveTransformationRule(List.of("b1"), List.of("c1", "f"))); rules.add(new DeleteTransformationRule(List.of("e"))); - SchemaEventTransformer eventTransformer = new SchemaEventTransformer(rules); - - Map result = eventTransformer.transform(event); - + for (var rule : rules) { + result = rule.apply(result); + } assertEquals(2, result.keySet().size()); assertTrue(result.containsKey("a1")); diff --git a/streampipes-connect-shared/src/test/java/org/apache/streampipes/connect/shared/preprocessing/transform/value/CorrectionValueTest.java b/streampipes-connect-shared/src/test/java/org/apache/streampipes/connect/shared/preprocessing/transform/value/CorrectionValueTest.java index a8605ef579..5da81a9a24 100644 --- a/streampipes-connect-shared/src/test/java/org/apache/streampipes/connect/shared/preprocessing/transform/value/CorrectionValueTest.java +++ b/streampipes-connect-shared/src/test/java/org/apache/streampipes/connect/shared/preprocessing/transform/value/CorrectionValueTest.java @@ -68,7 +68,7 @@ public void testAdd() { "ADD" ); - var resultEvent = correctionRule.transform(event); + var resultEvent = correctionRule.apply(event); assertNotNull(resultEvent); assertEquals(110.0, resultEvent.get(propertyNameBasicValue)); } @@ -81,7 +81,7 @@ public void testSubtract() { 10.0, "SUBTRACT" ); - var resultEvent = correctionRule.transform(event); + var resultEvent = correctionRule.apply(event); assertNotNull(resultEvent); assertEquals(90.0, resultEvent.get(propertyNameBasicValue)); } @@ -94,7 +94,7 @@ public void testMultiply() { 1.5, "MULTIPLY" ); - var resultEvent = correctionRule.transform(event); + var resultEvent = correctionRule.apply(event); assertNotNull(resultEvent); assertEquals(150.0, resultEvent.get(propertyNameBasicValue)); } @@ -107,7 +107,7 @@ public void testDivide() { 5, "DIVIDE" ); - var resultEvent = correctionRule.transform(event); + var resultEvent = correctionRule.apply(event); assertNotNull(resultEvent); assertEquals(20.0, resultEvent.get(propertyNameBasicValue)); } @@ -120,7 +120,7 @@ public void testDivideByZero() { 0.0, "DIVIDE" ); - var resultEvent = correctionRule.transform(event); + var resultEvent = correctionRule.apply(event); assertNotNull(resultEvent); assertEquals(Double.POSITIVE_INFINITY, resultEvent.get(propertyNameBasicValue)); } @@ -136,7 +136,7 @@ public void testNonNumericValue() { assertThrows ( String.format("Selected property `%s` does not contain a numeric value: `%s", propertyNameOtherValue, "Hello"), RuntimeException.class, - () -> correctionRule.transform(event).get(propertyNameOtherValue) + () -> correctionRule.apply(event).get(propertyNameOtherValue) ); @@ -150,7 +150,7 @@ public void testUnsupportedOperation() { 10.0, "TEST" ); - var resultEvent = correctionRule.transform(event); + var resultEvent = correctionRule.apply(event); assertNotNull(resultEvent); assertEquals(100.0, resultEvent.get(propertyNameBasicValue)); } diff --git a/streampipes-connect-shared/src/test/java/org/apache/streampipes/connect/shared/preprocessing/transform/value/UnitTransformRuleTest.java b/streampipes-connect-shared/src/test/java/org/apache/streampipes/connect/shared/preprocessing/transform/value/UnitTransformRuleTest.java index 47a507f878..cedad9837d 100644 --- a/streampipes-connect-shared/src/test/java/org/apache/streampipes/connect/shared/preprocessing/transform/value/UnitTransformRuleTest.java +++ b/streampipes-connect-shared/src/test/java/org/apache/streampipes/connect/shared/preprocessing/transform/value/UnitTransformRuleTest.java @@ -60,7 +60,7 @@ public void transformList() { UnitTransformationRule unitTransformationRule = new UnitTransformationRule(keys, "http://qudt.org/vocab/unit#DegreeCelsius", "http://qudt.org/vocab/unit#Kelvin"); - var result = unitTransformationRule.transform(event); + var result = unitTransformationRule.apply(event); assertEquals(1, result.keySet().size()); assertEquals(273.15, @@ -93,7 +93,7 @@ public void transformNested() { UnitTransformationRule unitTransformationRule = new UnitTransformationRule(keys, "http://qudt.org/vocab/unit#DegreeCelsius", "http://qudt.org/vocab/unit#Kelvin"); - var result = unitTransformationRule.transform(event); + var result = unitTransformationRule.apply(event); assertEquals(1, result.keySet().size()); assertEquals(283.15, @@ -123,7 +123,7 @@ public void transformMultiEvent() { event.put("value1", 0.0); event.put("value2", 10.0); - var result = unitTransformationRule.transform(event); + var result = unitTransformationRule.apply(event); assertEquals(2, result.keySet().size()); assertEquals(283.15, result.get(eventPropertyValue2.getLabel())); @@ -132,7 +132,7 @@ public void transformMultiEvent() { event.put("value1", 20.0); event.put("value2", 20.0); - result = unitTransformationRule.transform(event); + result = unitTransformationRule.apply(event); assertEquals(2, result.keySet().size()); assertEquals(293.15, result.get(eventPropertyValue2.getRuntimeName())); @@ -141,7 +141,7 @@ public void transformMultiEvent() { event.put("value1", 0.0); event.put("value2", 0.0); - result = unitTransformationRule.transform(event); + result = unitTransformationRule.apply(event); assertEquals(2, result.keySet().size()); assertEquals(273.15, result.get(eventPropertyValue2.getRuntimeName())); } diff --git a/streampipes-connect-shared/src/test/java/org/apache/streampipes/connect/shared/preprocessing/transform/value/ValueEventTransformerTest.java b/streampipes-connect-shared/src/test/java/org/apache/streampipes/connect/shared/preprocessing/transform/value/ValueEventTransformerTest.java index 13ae88520f..92c370378f 100644 --- a/streampipes-connect-shared/src/test/java/org/apache/streampipes/connect/shared/preprocessing/transform/value/ValueEventTransformerTest.java +++ b/streampipes-connect-shared/src/test/java/org/apache/streampipes/connect/shared/preprocessing/transform/value/ValueEventTransformerTest.java @@ -18,6 +18,7 @@ package org.apache.streampipes.connect.shared.preprocessing.transform.value; +import org.apache.streampipes.connect.shared.preprocessing.transform.TransformationRule; import org.apache.streampipes.model.schema.EventProperty; import org.apache.streampipes.model.schema.EventPropertyPrimitive; import org.apache.streampipes.model.schema.EventSchema; @@ -48,14 +49,15 @@ public void transform() { List keys = new ArrayList<>(); keys.add("a"); - List rules = new ArrayList<>(); + List rules = new ArrayList<>(); rules.add(new UnitTransformationRule(keys, "http://qudt.org/vocab/unit#Kelvin", "http://qudt.org/vocab/unit#DegreeCelsius")); - ValueEventTransformer eventTransformer = new ValueEventTransformer(rules); - Map result = eventTransformer.transform(event); + for (var rule: rules) { + event = rule.apply(event); + } - assertEquals(0.0, result.get(eventPropertyf.getRuntimeName())); + assertEquals(0.0, event.get(eventPropertyf.getRuntimeName())); } diff --git a/streampipes-extensions-management/src/main/java/org/apache/streampipes/extensions/management/connect/adapter/AdapterPipelineGenerator.java b/streampipes-extensions-management/src/main/java/org/apache/streampipes/extensions/management/connect/adapter/AdapterPipelineGenerator.java index 5f0ae11026..3fe2452559 100644 --- a/streampipes-extensions-management/src/main/java/org/apache/streampipes/extensions/management/connect/adapter/AdapterPipelineGenerator.java +++ b/streampipes-extensions-management/src/main/java/org/apache/streampipes/extensions/management/connect/adapter/AdapterPipelineGenerator.java @@ -19,8 +19,6 @@ package org.apache.streampipes.extensions.management.connect.adapter; import org.apache.streampipes.connect.shared.AdapterPipelineGeneratorBase; -import org.apache.streampipes.connect.shared.preprocessing.elements.TransformStreamAdapterElement; -import org.apache.streampipes.connect.shared.preprocessing.transform.stream.DuplicateFilterPipelineElement; import org.apache.streampipes.extensions.management.client.StreamPipesClientResolver; import org.apache.streampipes.extensions.management.connect.adapter.model.pipeline.AdapterPipeline; import org.apache.streampipes.extensions.management.connect.adapter.preprocessing.elements.SendToBrokerAdapterSink; @@ -41,32 +39,16 @@ public class AdapterPipelineGenerator extends AdapterPipelineGeneratorBase { public AdapterPipeline generatePipeline(AdapterDescription adapterDescription) { - var pipelineElements = makeAdapterPipelineElements(adapterDescription.getRules()); + var pipelineElements = makeAdapterPipelineElements(adapterDescription.getRules(), true); - var duplicatesTransformationRuleDescription = - getRemoveDuplicateRule(adapterDescription.getRules()); - if (duplicatesTransformationRuleDescription != null) { - pipelineElements.add( - new DuplicateFilterPipelineElement(duplicatesTransformationRuleDescription.getFilterTimeWindow())); - } - - var transformStreamAdapterElement = new TransformStreamAdapterElement(); - var eventRateTransformationRuleDescription = getEventRateTransformationRule(adapterDescription.getRules()); - if (eventRateTransformationRuleDescription != null) { - transformStreamAdapterElement.addStreamTransformationRuleDescription(eventRateTransformationRuleDescription); - } - pipelineElements.add(transformStreamAdapterElement); - - if (adapterDescription.getEventGrounding() != null - && adapterDescription.getEventGrounding().getTransportProtocol() != null - && adapterDescription.getEventGrounding().getTransportProtocol().getBrokerHostname() != null) { + if (hasValidGrounding(adapterDescription)) { return new AdapterPipeline( pipelineElements, getAdapterSink(adapterDescription), adapterDescription.getEventSchema()); + } else { + return new AdapterPipeline(pipelineElements, adapterDescription.getEventSchema()); } - - return new AdapterPipeline(pipelineElements, adapterDescription.getEventSchema()); } private SendToBrokerAdapterSink getAdapterSink(AdapterDescription adapterDescription) { @@ -95,4 +77,10 @@ private MessagingSettings getMessagingSettings() { var client = new StreamPipesClientResolver().makeStreamPipesClientInstance(); return client.adminApi().getMessagingSettings(); } + + private boolean hasValidGrounding(AdapterDescription adapterDescription) { + return adapterDescription.getEventGrounding() != null + && adapterDescription.getEventGrounding().getTransportProtocol() != null + && adapterDescription.getEventGrounding().getTransportProtocol().getBrokerHostname() != null; + } } diff --git a/streampipes-extensions/streampipes-connect-adapters/src/test/java/org/apache/streampipes/connect/adapters/generic/elements/AddTimestampTest.java b/streampipes-extensions/streampipes-connect-adapters/src/test/java/org/apache/streampipes/connect/adapters/generic/elements/AddTimestampTest.java index 283e5862c1..0791808c6e 100644 --- a/streampipes-extensions/streampipes-connect-adapters/src/test/java/org/apache/streampipes/connect/adapters/generic/elements/AddTimestampTest.java +++ b/streampipes-extensions/streampipes-connect-adapters/src/test/java/org/apache/streampipes/connect/adapters/generic/elements/AddTimestampTest.java @@ -18,7 +18,7 @@ package org.apache.streampipes.connect.adapters.generic.elements; -import org.apache.streampipes.connect.shared.preprocessing.elements.AddTimestampPipelineElement; +import org.apache.streampipes.connect.shared.preprocessing.transform.value.AddTimestampTransformationRule; import org.junit.Assert; import org.junit.Test; @@ -31,9 +31,9 @@ public class AddTimestampTest { @Test public void addTimestamp() { - AddTimestampPipelineElement addTimestamp = new AddTimestampPipelineElement("timestamp"); + var addTimestamp = new AddTimestampTransformationRule("timestamp"); Map event = new HashMap<>(); - event = addTimestamp.process(event); + event = addTimestamp.apply(event); Assert.assertNotNull(event.get("timestamp")); } diff --git a/streampipes-extensions/streampipes-connect-adapters/src/test/java/org/apache/streampipes/connect/adapters/generic/elements/DuplicateFilterTest.java b/streampipes-extensions/streampipes-connect-adapters/src/test/java/org/apache/streampipes/connect/adapters/generic/elements/DuplicateFilterTest.java index 0c88d383cd..691c38e62d 100644 --- a/streampipes-extensions/streampipes-connect-adapters/src/test/java/org/apache/streampipes/connect/adapters/generic/elements/DuplicateFilterTest.java +++ b/streampipes-extensions/streampipes-connect-adapters/src/test/java/org/apache/streampipes/connect/adapters/generic/elements/DuplicateFilterTest.java @@ -35,27 +35,27 @@ public class DuplicateFilterTest { @Test public void duplicateSimple() { DuplicateFilterPipelineElement duplicateFilter = new DuplicateFilterPipelineElement("0"); - List events = generateEvents(); + List> events = generateEvents(); - assertNotNull(duplicateFilter.process(events.get(0))); - assertNull(duplicateFilter.process(events.get(0))); + assertNotNull(duplicateFilter.apply(events.get(0))); + assertNull(duplicateFilter.apply(events.get(0))); } @Test public void duplicateComplex() { DuplicateFilterPipelineElement duplicateFilter = new DuplicateFilterPipelineElement("0"); - List events = generateEvents(); - - assertNotNull(duplicateFilter.process(events.get(0))); - assertNotNull(duplicateFilter.process(events.get(1))); - assertNotNull(duplicateFilter.process(events.get(2))); - assertNotNull(duplicateFilter.process(events.get(3))); - assertNotNull(duplicateFilter.process(events.get(4))); - assertNotNull(duplicateFilter.process(events.get(5))); - assertNotNull(duplicateFilter.process(events.get(6))); - assertNotNull(duplicateFilter.process(events.get(7))); - assertNull(duplicateFilter.process(events.get(0))); + List> events = generateEvents(); + + assertNotNull(duplicateFilter.apply(events.get(0))); + assertNotNull(duplicateFilter.apply(events.get(1))); + assertNotNull(duplicateFilter.apply(events.get(2))); + assertNotNull(duplicateFilter.apply(events.get(3))); + assertNotNull(duplicateFilter.apply(events.get(4))); + assertNotNull(duplicateFilter.apply(events.get(5))); + assertNotNull(duplicateFilter.apply(events.get(6))); + assertNotNull(duplicateFilter.apply(events.get(7))); + assertNull(duplicateFilter.apply(events.get(0))); } @@ -106,8 +106,8 @@ public void CleanUp2() throws InterruptedException { */ - private List generateEvents() { - List list = new LinkedList(); + private List> generateEvents() { + List> list = new LinkedList<>(); list.add(makeMap("Test", 123)); list.add(makeMap("Test", 1234)); @@ -123,7 +123,7 @@ private List generateEvents() { } private Map makeMap(String key, Object value) { - Map map = new LinkedHashMap(); + Map map = new LinkedHashMap<>(); map.put(key, value); return map; } diff --git a/streampipes-model/src/main/java/org/apache/streampipes/model/connect/rules/ITransformationRuleVisitor.java b/streampipes-model/src/main/java/org/apache/streampipes/model/connect/rules/ITransformationRuleVisitor.java new file mode 100644 index 0000000000..bb45ff0eaf --- /dev/null +++ b/streampipes-model/src/main/java/org/apache/streampipes/model/connect/rules/ITransformationRuleVisitor.java @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.streampipes.model.connect.rules; + +import org.apache.streampipes.model.connect.rules.schema.CreateNestedRuleDescription; +import org.apache.streampipes.model.connect.rules.schema.DeleteRuleDescription; +import org.apache.streampipes.model.connect.rules.schema.MoveRuleDescription; +import org.apache.streampipes.model.connect.rules.schema.RenameRuleDescription; +import org.apache.streampipes.model.connect.rules.stream.EventRateTransformationRuleDescription; +import org.apache.streampipes.model.connect.rules.stream.RemoveDuplicatesTransformationRuleDescription; +import org.apache.streampipes.model.connect.rules.value.AddTimestampRuleDescription; +import org.apache.streampipes.model.connect.rules.value.AddValueTransformationRuleDescription; +import org.apache.streampipes.model.connect.rules.value.ChangeDatatypeTransformationRuleDescription; +import org.apache.streampipes.model.connect.rules.value.CorrectionValueTransformationRuleDescription; +import org.apache.streampipes.model.connect.rules.value.TimestampTranfsformationRuleDescription; +import org.apache.streampipes.model.connect.rules.value.UnitTransformRuleDescription; + +public interface ITransformationRuleVisitor { + + void visit(CreateNestedRuleDescription rule); + + void visit(DeleteRuleDescription rule); + + void visit(MoveRuleDescription rule); + + void visit(RenameRuleDescription rule); + + void visit(EventRateTransformationRuleDescription rule); + + void visit(RemoveDuplicatesTransformationRuleDescription rule); + + void visit(AddTimestampRuleDescription rule); + + void visit(AddValueTransformationRuleDescription rule); + + void visit(ChangeDatatypeTransformationRuleDescription rule); + + void visit(CorrectionValueTransformationRuleDescription rule); + + void visit(TimestampTranfsformationRuleDescription rule); + + void visit(UnitTransformRuleDescription rule); +} diff --git a/streampipes-model/src/main/java/org/apache/streampipes/model/connect/rules/TransformationRuleDescription.java b/streampipes-model/src/main/java/org/apache/streampipes/model/connect/rules/TransformationRuleDescription.java index 25787af6d2..0ea2194456 100644 --- a/streampipes-model/src/main/java/org/apache/streampipes/model/connect/rules/TransformationRuleDescription.java +++ b/streampipes-model/src/main/java/org/apache/streampipes/model/connect/rules/TransformationRuleDescription.java @@ -57,4 +57,8 @@ public abstract class TransformationRuleDescription { public TransformationRuleDescription() { super(); } + + public abstract void accept(ITransformationRuleVisitor visitor); + + public abstract int getRulePriority(); } diff --git a/streampipes-model/src/main/java/org/apache/streampipes/model/connect/rules/schema/CreateNestedRuleDescription.java b/streampipes-model/src/main/java/org/apache/streampipes/model/connect/rules/schema/CreateNestedRuleDescription.java index d065e0bc87..cb25187c1e 100644 --- a/streampipes-model/src/main/java/org/apache/streampipes/model/connect/rules/schema/CreateNestedRuleDescription.java +++ b/streampipes-model/src/main/java/org/apache/streampipes/model/connect/rules/schema/CreateNestedRuleDescription.java @@ -18,6 +18,8 @@ package org.apache.streampipes.model.connect.rules.schema; +import org.apache.streampipes.model.connect.rules.ITransformationRuleVisitor; + public class CreateNestedRuleDescription extends SchemaTransformationRuleDescription { private String runtimeKey; @@ -43,4 +45,14 @@ public String getRuntimeKey() { public void setRuntimeKey(String runtimeKey) { this.runtimeKey = runtimeKey; } + + @Override + public void accept(ITransformationRuleVisitor visitor) { + visitor.visit(this); + } + + @Override + public int getRulePriority() { + return 230; + } } diff --git a/streampipes-model/src/main/java/org/apache/streampipes/model/connect/rules/schema/DeleteRuleDescription.java b/streampipes-model/src/main/java/org/apache/streampipes/model/connect/rules/schema/DeleteRuleDescription.java index 219cd84a94..5532dfc358 100644 --- a/streampipes-model/src/main/java/org/apache/streampipes/model/connect/rules/schema/DeleteRuleDescription.java +++ b/streampipes-model/src/main/java/org/apache/streampipes/model/connect/rules/schema/DeleteRuleDescription.java @@ -18,6 +18,8 @@ package org.apache.streampipes.model.connect.rules.schema; +import org.apache.streampipes.model.connect.rules.ITransformationRuleVisitor; + public class DeleteRuleDescription extends SchemaTransformationRuleDescription { private String runtimeKey; @@ -44,4 +46,14 @@ public String getRuntimeKey() { public void setRuntimeKey(String runtimeKey) { this.runtimeKey = runtimeKey; } + + @Override + public void accept(ITransformationRuleVisitor visitor) { + visitor.visit(this); + } + + @Override + public int getRulePriority() { + return 240; + } } diff --git a/streampipes-model/src/main/java/org/apache/streampipes/model/connect/rules/schema/MoveRuleDescription.java b/streampipes-model/src/main/java/org/apache/streampipes/model/connect/rules/schema/MoveRuleDescription.java index 312d165795..468d238cb5 100644 --- a/streampipes-model/src/main/java/org/apache/streampipes/model/connect/rules/schema/MoveRuleDescription.java +++ b/streampipes-model/src/main/java/org/apache/streampipes/model/connect/rules/schema/MoveRuleDescription.java @@ -18,6 +18,8 @@ package org.apache.streampipes.model.connect.rules.schema; +import org.apache.streampipes.model.connect.rules.ITransformationRuleVisitor; + public class MoveRuleDescription extends SchemaTransformationRuleDescription { private String oldRuntimeKey; @@ -55,5 +57,15 @@ public String getNewRuntimeKey() { public void setNewRuntimeKey(String newRuntimeKey) { this.newRuntimeKey = newRuntimeKey; } + + @Override + public void accept(ITransformationRuleVisitor visitor) { + visitor.visit(this); + } + + @Override + public int getRulePriority() { + return 220; + } } diff --git a/streampipes-model/src/main/java/org/apache/streampipes/model/connect/rules/schema/RenameRuleDescription.java b/streampipes-model/src/main/java/org/apache/streampipes/model/connect/rules/schema/RenameRuleDescription.java index b6ccf443ae..2d61e53507 100644 --- a/streampipes-model/src/main/java/org/apache/streampipes/model/connect/rules/schema/RenameRuleDescription.java +++ b/streampipes-model/src/main/java/org/apache/streampipes/model/connect/rules/schema/RenameRuleDescription.java @@ -18,6 +18,8 @@ package org.apache.streampipes.model.connect.rules.schema; +import org.apache.streampipes.model.connect.rules.ITransformationRuleVisitor; + public class RenameRuleDescription extends SchemaTransformationRuleDescription { private String oldRuntimeKey; @@ -55,4 +57,14 @@ public String getNewRuntimeKey() { public void setNewRuntimeKey(String newRuntimeKey) { this.newRuntimeKey = newRuntimeKey; } + + @Override + public void accept(ITransformationRuleVisitor visitor) { + visitor.visit(this); + } + + @Override + public int getRulePriority() { + return 210; + } } diff --git a/streampipes-model/src/main/java/org/apache/streampipes/model/connect/rules/stream/EventRateTransformationRuleDescription.java b/streampipes-model/src/main/java/org/apache/streampipes/model/connect/rules/stream/EventRateTransformationRuleDescription.java index 8d73a3d4c0..a681eddb42 100644 --- a/streampipes-model/src/main/java/org/apache/streampipes/model/connect/rules/stream/EventRateTransformationRuleDescription.java +++ b/streampipes-model/src/main/java/org/apache/streampipes/model/connect/rules/stream/EventRateTransformationRuleDescription.java @@ -18,6 +18,8 @@ package org.apache.streampipes.model.connect.rules.stream; +import org.apache.streampipes.model.connect.rules.ITransformationRuleVisitor; + public class EventRateTransformationRuleDescription extends StreamTransformationRuleDescription { private long aggregationTimeWindow; @@ -51,4 +53,14 @@ public String getAggregationType() { public void setAggregationType(String aggregationTypes) { this.aggregationType = aggregationTypes; } + + @Override + public void accept(ITransformationRuleVisitor visitor) { + visitor.visit(this); + } + + @Override + public int getRulePriority() { + return 420; + } } diff --git a/streampipes-model/src/main/java/org/apache/streampipes/model/connect/rules/stream/RemoveDuplicatesTransformationRuleDescription.java b/streampipes-model/src/main/java/org/apache/streampipes/model/connect/rules/stream/RemoveDuplicatesTransformationRuleDescription.java index 0c67496d42..ddbe5fb211 100644 --- a/streampipes-model/src/main/java/org/apache/streampipes/model/connect/rules/stream/RemoveDuplicatesTransformationRuleDescription.java +++ b/streampipes-model/src/main/java/org/apache/streampipes/model/connect/rules/stream/RemoveDuplicatesTransformationRuleDescription.java @@ -18,6 +18,8 @@ package org.apache.streampipes.model.connect.rules.stream; +import org.apache.streampipes.model.connect.rules.ITransformationRuleVisitor; + public class RemoveDuplicatesTransformationRuleDescription extends StreamTransformationRuleDescription { private String filterTimeWindow; @@ -38,4 +40,14 @@ public String getFilterTimeWindow() { public void setFilterTimeWindow(String filterTimeWindow) { this.filterTimeWindow = filterTimeWindow; } + + @Override + public void accept(ITransformationRuleVisitor visitor) { + visitor.visit(this); + } + + @Override + public int getRulePriority() { + return 410; + } } diff --git a/streampipes-model/src/main/java/org/apache/streampipes/model/connect/rules/value/AddTimestampRuleDescription.java b/streampipes-model/src/main/java/org/apache/streampipes/model/connect/rules/value/AddTimestampRuleDescription.java index 66f8f13837..c86362c16e 100644 --- a/streampipes-model/src/main/java/org/apache/streampipes/model/connect/rules/value/AddTimestampRuleDescription.java +++ b/streampipes-model/src/main/java/org/apache/streampipes/model/connect/rules/value/AddTimestampRuleDescription.java @@ -18,6 +18,8 @@ package org.apache.streampipes.model.connect.rules.value; +import org.apache.streampipes.model.connect.rules.ITransformationRuleVisitor; + public class AddTimestampRuleDescription extends ValueTransformationRuleDescription { private String runtimeKey; @@ -42,4 +44,14 @@ public String getRuntimeKey() { public void setRuntimeKey(String runtimeKey) { this.runtimeKey = runtimeKey; } + + @Override + public void accept(ITransformationRuleVisitor visitor) { + visitor.visit(this); + } + + @Override + public int getRulePriority() { + return 100; + } } diff --git a/streampipes-model/src/main/java/org/apache/streampipes/model/connect/rules/value/AddValueTransformationRuleDescription.java b/streampipes-model/src/main/java/org/apache/streampipes/model/connect/rules/value/AddValueTransformationRuleDescription.java index d2adb10942..de1aa86ec6 100644 --- a/streampipes-model/src/main/java/org/apache/streampipes/model/connect/rules/value/AddValueTransformationRuleDescription.java +++ b/streampipes-model/src/main/java/org/apache/streampipes/model/connect/rules/value/AddValueTransformationRuleDescription.java @@ -18,6 +18,8 @@ package org.apache.streampipes.model.connect.rules.value; +import org.apache.streampipes.model.connect.rules.ITransformationRuleVisitor; + public class AddValueTransformationRuleDescription extends ValueTransformationRuleDescription { private String runtimeKey; @@ -49,4 +51,14 @@ public String getStaticValue() { public void setStaticValue(String staticValue) { this.staticValue = staticValue; } + + @Override + public void accept(ITransformationRuleVisitor visitor) { + visitor.visit(this); + } + + @Override + public int getRulePriority() { + return 110; + } } diff --git a/streampipes-model/src/main/java/org/apache/streampipes/model/connect/rules/value/ChangeDatatypeTransformationRuleDescription.java b/streampipes-model/src/main/java/org/apache/streampipes/model/connect/rules/value/ChangeDatatypeTransformationRuleDescription.java index aa64e647bf..427cb39c50 100644 --- a/streampipes-model/src/main/java/org/apache/streampipes/model/connect/rules/value/ChangeDatatypeTransformationRuleDescription.java +++ b/streampipes-model/src/main/java/org/apache/streampipes/model/connect/rules/value/ChangeDatatypeTransformationRuleDescription.java @@ -18,6 +18,8 @@ package org.apache.streampipes.model.connect.rules.value; +import org.apache.streampipes.model.connect.rules.ITransformationRuleVisitor; + public class ChangeDatatypeTransformationRuleDescription extends ValueTransformationRuleDescription { private String runtimeKey; @@ -57,4 +59,14 @@ public String getTargetDatatypeXsd() { public void setTargetDatatypeXsd(String targetDatatypeXsd) { this.targetDatatypeXsd = targetDatatypeXsd; } + + @Override + public void accept(ITransformationRuleVisitor visitor) { + visitor.visit(this); + } + + @Override + public int getRulePriority() { + return 340; + } } diff --git a/streampipes-model/src/main/java/org/apache/streampipes/model/connect/rules/value/CorrectionValueTransformationRuleDescription.java b/streampipes-model/src/main/java/org/apache/streampipes/model/connect/rules/value/CorrectionValueTransformationRuleDescription.java index 7747751211..304a9fe63a 100644 --- a/streampipes-model/src/main/java/org/apache/streampipes/model/connect/rules/value/CorrectionValueTransformationRuleDescription.java +++ b/streampipes-model/src/main/java/org/apache/streampipes/model/connect/rules/value/CorrectionValueTransformationRuleDescription.java @@ -18,6 +18,8 @@ package org.apache.streampipes.model.connect.rules.value; +import org.apache.streampipes.model.connect.rules.ITransformationRuleVisitor; + public class CorrectionValueTransformationRuleDescription extends ValueTransformationRuleDescription { private String runtimeKey; @@ -66,4 +68,14 @@ public String getOperator() { public void setOperator(String operator) { this.operator = operator; } + + @Override + public void accept(ITransformationRuleVisitor visitor) { + visitor.visit(this); + } + + @Override + public int getRulePriority() { + return 330; + } } diff --git a/streampipes-model/src/main/java/org/apache/streampipes/model/connect/rules/value/TimestampTranfsformationRuleDescription.java b/streampipes-model/src/main/java/org/apache/streampipes/model/connect/rules/value/TimestampTranfsformationRuleDescription.java index dd19158fca..07bbd473c3 100644 --- a/streampipes-model/src/main/java/org/apache/streampipes/model/connect/rules/value/TimestampTranfsformationRuleDescription.java +++ b/streampipes-model/src/main/java/org/apache/streampipes/model/connect/rules/value/TimestampTranfsformationRuleDescription.java @@ -18,6 +18,8 @@ package org.apache.streampipes.model.connect.rules.value; +import org.apache.streampipes.model.connect.rules.ITransformationRuleVisitor; + public class TimestampTranfsformationRuleDescription extends ValueTransformationRuleDescription { private String runtimeKey; @@ -79,4 +81,14 @@ public long getMultiplier() { public void setMultiplier(long multiplier) { this.multiplier = multiplier; } + + @Override + public void accept(ITransformationRuleVisitor visitor) { + visitor.visit(this); + } + + @Override + public int getRulePriority() { + return 320; + } } diff --git a/streampipes-model/src/main/java/org/apache/streampipes/model/connect/rules/value/TimestampTransformationRuleMode.java b/streampipes-model/src/main/java/org/apache/streampipes/model/connect/rules/value/TimestampTransformationRuleMode.java deleted file mode 100644 index fbec7e64d2..0000000000 --- a/streampipes-model/src/main/java/org/apache/streampipes/model/connect/rules/value/TimestampTransformationRuleMode.java +++ /dev/null @@ -1,26 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package org.apache.streampipes.model.connect.rules.value; - - -public class TimestampTransformationRuleMode { - - public static final String FORMAT_STRING = "formatString"; - public static final String TIME_UNIT = "timeUnit"; -} diff --git a/streampipes-model/src/main/java/org/apache/streampipes/model/connect/rules/value/UnitTransformRuleDescription.java b/streampipes-model/src/main/java/org/apache/streampipes/model/connect/rules/value/UnitTransformRuleDescription.java index ae5c56cfb7..5a6c0707c8 100644 --- a/streampipes-model/src/main/java/org/apache/streampipes/model/connect/rules/value/UnitTransformRuleDescription.java +++ b/streampipes-model/src/main/java/org/apache/streampipes/model/connect/rules/value/UnitTransformRuleDescription.java @@ -18,6 +18,8 @@ package org.apache.streampipes.model.connect.rules.value; +import org.apache.streampipes.model.connect.rules.ITransformationRuleVisitor; + public class UnitTransformRuleDescription extends ValueTransformationRuleDescription { private String runtimeKey; @@ -67,4 +69,14 @@ public String getToUnitRessourceURL() { public void setToUnitRessourceURL(String toUnitRessourceURL) { this.toUnitRessourceURL = toUnitRessourceURL; } + + @Override + public void accept(ITransformationRuleVisitor visitor) { + visitor.visit(this); + } + + @Override + public int getRulePriority() { + return 310; + } } diff --git a/ui/src/app/connect/components/adapter-configuration/schema-editor/event-property-row/event-property-row.component.ts b/ui/src/app/connect/components/adapter-configuration/schema-editor/event-property-row/event-property-row.component.ts index adf210c24b..feffca54bd 100644 --- a/ui/src/app/connect/components/adapter-configuration/schema-editor/event-property-row/event-property-row.component.ts +++ b/ui/src/app/connect/components/adapter-configuration/schema-editor/event-property-row/event-property-row.component.ts @@ -78,17 +78,20 @@ export class EventPropertyRowComponent implements OnInit { const originalProperty = this.findOriginalProperty( this.originalEventSchema.eventProperties, ); - this.originalRuntimeName = originalProperty.runtimeName; - this.showFieldStatus = - this.fieldStatusInfo && - this.fieldStatusInfo[this.originalRuntimeName] !== undefined; - if (this.isPrimitive) { - this.originalRuntimeType = this.parseType( - originalProperty.runtimeType, - ); - this.runtimeType = this.parseType( - (this.node.data as EventPropertyPrimitive).runtimeType, - ); + if (originalProperty) { + this.originalRuntimeName = originalProperty.runtimeName; + this.showFieldStatus = + this.fieldStatusInfo && + this.fieldStatusInfo[this.originalRuntimeName] !== + undefined; + if (this.isPrimitive) { + this.originalRuntimeType = this.parseType( + originalProperty.runtimeType, + ); + this.runtimeType = this.parseType( + (this.node.data as EventPropertyPrimitive).runtimeType, + ); + } } } diff --git a/ui/src/app/connect/components/adapter-configuration/schema-editor/event-schema/event-schema.component.ts b/ui/src/app/connect/components/adapter-configuration/schema-editor/event-schema/event-schema.component.ts index ca0a4da551..d9a549d183 100644 --- a/ui/src/app/connect/components/adapter-configuration/schema-editor/event-schema/event-schema.component.ts +++ b/ui/src/app/connect/components/adapter-configuration/schema-editor/event-schema/event-schema.component.ts @@ -201,6 +201,7 @@ export class EventSchemaComponent implements OnChanges { nested.elementId = uuid; nested.eventProperties = []; nested.domainProperties = []; + nested.runtimeName = 'nested'; if (!eventProperty) { this.eventSchema.eventProperties.push(nested); } else { diff --git a/ui/src/app/connect/dialog/edit-event-property/components/edit-schema-transformation/edit-schema-transformation.component.html b/ui/src/app/connect/dialog/edit-event-property/components/edit-schema-transformation/edit-schema-transformation.component.html index 8ca237642b..b5138f34ab 100644 --- a/ui/src/app/connect/dialog/edit-event-property/components/edit-schema-transformation/edit-schema-transformation.component.html +++ b/ui/src/app/connect/dialog/edit-event-property/components/edit-schema-transformation/edit-schema-transformation.component.html @@ -89,7 +89,7 @@ Date: Wed, 4 Oct 2023 21:14:14 +0200 Subject: [PATCH 4/4] Add comment to skipped rules in stateful transformation visitor --- ...fulTransformationRuleGeneratorVisitor.java | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/generator/StatefulTransformationRuleGeneratorVisitor.java b/streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/generator/StatefulTransformationRuleGeneratorVisitor.java index 059cd4920d..c8e3c4cb5d 100644 --- a/streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/generator/StatefulTransformationRuleGeneratorVisitor.java +++ b/streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/generator/StatefulTransformationRuleGeneratorVisitor.java @@ -37,22 +37,22 @@ public class StatefulTransformationRuleGeneratorVisitor extends TransformationRu @Override public void visit(CreateNestedRuleDescription rule) { - + // skip (not a stateful transformation) } @Override public void visit(DeleteRuleDescription rule) { - + // skip (not a stateful transformation) } @Override public void visit(MoveRuleDescription rule) { - + // skip (not a stateful transformation) } @Override public void visit(RenameRuleDescription rule) { - + // skip (not a stateful transformation) } @Override @@ -69,31 +69,31 @@ public void visit(RemoveDuplicatesTransformationRuleDescription ruleDesc) { @Override public void visit(AddTimestampRuleDescription rule) { - + // skip (not a stateful transformation) } @Override public void visit(AddValueTransformationRuleDescription rule) { - + // skip (not a stateful transformation) } @Override public void visit(ChangeDatatypeTransformationRuleDescription rule) { - + // skip (not a stateful transformation) } @Override public void visit(CorrectionValueTransformationRuleDescription rule) { - + // skip (not a stateful transformation) } @Override public void visit(TimestampTranfsformationRuleDescription rule) { - + // skip (not a stateful transformation) } @Override public void visit(UnitTransformRuleDescription rule) { - + // skip (not a stateful transformation) } }