From 359b86308df68c6c53832f93883a9d04563cc445 Mon Sep 17 00:00:00 2001 From: Dominik Riemer Date: Fri, 11 Oct 2024 16:17:29 +0200 Subject: [PATCH] feat(#3280): Add support for pipeline templates (#3281) * feat(#3266): Add API to programmatically create pipelines * Fix checkstyle * feat(#3280): Add support for pipeline templates * Fix validation * Remove obsolete stylesheet * Fix dependency * Add default template to setup * feat(#3292): Show adapter and pipeline configuration in UI (#3293) * feat(#3292): Add code tab to adapter view * Show pipeline code in UI * Add header * Fix checkstyle * Fix checkstyle * Fix template visitor * Remove empty css class --- .../compact/PersistPipelineHandler.java | 113 +++--- .../AdapterEnrichmentRuleGenerator.java | 4 +- .../generator/CompactAdapterGenerator.java | 123 +++++++ .../management/CompactAdapterManagement.java | 17 + .../adapter/compact/CompactAdapter.java | 3 + .../adapter/compact/CompactEventProperty.java | 3 + .../adapter/compact/CreateOptions.java | 7 +- .../pipeline/compact/CompactPipeline.java | 2 + .../compact/CompactPipelineElement.java | 3 + .../model/template/BoundPipelineElement.java | 57 --- .../template/CompactPipelineTemplate.java | 106 ++++++ .../PipelineElementTemplateConfig.java | 67 ---- .../template/PipelinePlaceholderConfig.java | 23 ++ ...ntainer.java => PipelinePlaceholders.java} | 25 +- .../template/PipelineTemplateDescription.java | 87 ----- .../PipelineTemplateGenerationRequest.java | 30 ++ .../template/PipelineTemplateInvocation.java | 97 ------ .../apache/streampipes/model/util/Cloner.java | 8 - .../compact/CompactPipelineManagement.java | 14 + .../generation/CompactPipelineConverter.java | 85 +++++ .../InvocablePipelineElementGenerator.java | 2 +- .../PipelineElementConfigurationStep.java | 12 +- .../setup/CouchDbInstallationStep.java | 5 + .../AddDefaultPipelineTemplatesTask.java | 37 ++ .../template/CompactConfigGenerator.java | 211 ++++++++++++ .../PipelineElementTemplateVisitor.java | 43 +-- .../manager/template/PipelineGenerator.java | 139 -------- .../template/PipelineTemplateGenerator.java | 75 ---- .../PipelineTemplateInvocationGenerator.java | 66 ---- .../PipelineTemplateInvocationHandler.java | 101 ------ .../template/PipelineTemplateManagement.java | 80 ----- .../CompactPipelineTemplateManagement.java | 98 ++++++ .../compact/MatchingStreamFinder.java | 77 +++++ .../instances/DataLakePipelineTemplate.java | 44 --- ...a => DefaultPipelineTemplateProvider.java} | 10 +- .../PersistDataLakePipelineTemplate.java | 70 ++++ .../rest/core/base/impl/CRUDResource.java | 29 +- .../rest/shared/constants/SpMediaType.java | 25 ++ .../rest/impl/PipelineResource.java | 21 ++ .../rest/impl/PipelineTemplate.java | 116 ++++--- .../rest/impl/connect/AdapterResource.java | 59 +++- .../impl/connect/CompactAdapterResource.java | 12 +- .../builder/BoundPipelineElementBuilder.java | 63 ---- .../sdk/builder/PipelineTemplateBuilder.java | 61 ---- .../core/migrations/AvailableMigrations.java | 4 +- .../AddDataLakePipelineTemplateMigration.java | 51 +++ .../storage/api/INoSqlStorage.java | 3 + .../couchdb/CouchDbStorageManager.java | 9 + .../StreamPipesNotificationSink.java | 2 +- .../src/lib/apis/adapter.service.ts | 10 + .../src/lib/apis/compact-pipeline.service.ts | 41 +++ .../src/lib/apis/pipeline-template.service.ts | 105 +----- .../src/lib/apis/pipeline.service.ts | 8 + .../src/lib/model/gen/streampipes-model.ts | 325 ++++++++++++++---- .../platform-services/src/public-api.ts | 1 + .../adapter-code-panel.component.html | 25 ++ .../adapter-code-panel.component.scss | 30 ++ .../adapter-code-panel.component.ts | 49 +++ .../adapter-options-panel.component.html | 22 +- ...start-adapter-configuration.component.html | 15 + .../start-adapter-configuration.component.ts | 1 + .../abstract-adapter-details.directive.ts | 2 + .../adapter-details-code.component.html | 38 ++ .../adapter-details-code.component.ts | 35 ++ .../adapter-details/adapter-details-tabs.ts | 5 + ui/src/app/connect/connect.module.ts | 10 +- .../adapter-started-dialog.component.ts | 106 +++--- .../configuration-code-panel.component.html | 34 ++ .../configuration-code-panel.component.scss | 30 ++ .../configuration-code-panel.component.ts | 32 ++ ui/src/app/core-ui/core-ui.module.ts | 9 + .../pipes}/json-pretty-print.pipe.ts | 0 .../core-ui/pipes/yaml-pretty-print.pipe.ts | 30 ++ .../pipeline-assembly-options.component.html | 18 +- .../pipeline-assembly-options.component.ts | 25 +- .../pipeline-assembly.component.html | 2 + .../pipeline-assembly.component.ts | 15 +- .../add-template-dialog.component.html | 41 +++ .../add-template-dialog.component.ts | 62 ++++ .../template-selection.component.html | 39 +++ .../template-selection.component.scss | 26 ++ .../template-selection.component.ts | 38 ++ .../save-pipeline-settings.component.html | 22 +- .../save-pipeline-settings.component.scss | 4 + .../save-pipeline-settings.component.ts | 16 +- ui/src/app/editor/editor.module.ts | 4 + .../services/pipeline-positioning.service.ts | 1 + .../pipeline-details-toolbar.component.html | 12 + .../pipeline-details-toolbar.component.ts | 3 + .../pipeline-code-dialog.component.html | 40 +++ .../pipeline-code-dialog.component.ts | 51 +++ .../pipeline-details.component.html | 1 + .../pipeline-details.component.ts | 15 + .../pipeline-details.module.ts | 2 + 94 files changed, 2435 insertions(+), 1364 deletions(-) create mode 100644 streampipes-connect-management/src/main/java/org/apache/streampipes/connect/management/compact/generator/CompactAdapterGenerator.java delete mode 100644 streampipes-model/src/main/java/org/apache/streampipes/model/template/BoundPipelineElement.java create mode 100644 streampipes-model/src/main/java/org/apache/streampipes/model/template/CompactPipelineTemplate.java delete mode 100644 streampipes-model/src/main/java/org/apache/streampipes/model/template/PipelineElementTemplateConfig.java create mode 100644 streampipes-model/src/main/java/org/apache/streampipes/model/template/PipelinePlaceholderConfig.java rename streampipes-model/src/main/java/org/apache/streampipes/model/template/{PipelineTemplateDescriptionContainer.java => PipelinePlaceholders.java} (61%) delete mode 100644 streampipes-model/src/main/java/org/apache/streampipes/model/template/PipelineTemplateDescription.java create mode 100644 streampipes-model/src/main/java/org/apache/streampipes/model/template/PipelineTemplateGenerationRequest.java delete mode 100644 streampipes-model/src/main/java/org/apache/streampipes/model/template/PipelineTemplateInvocation.java create mode 100644 streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/pipeline/compact/generation/CompactPipelineConverter.java create mode 100644 streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/setup/tasks/AddDefaultPipelineTemplatesTask.java create mode 100644 streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/template/CompactConfigGenerator.java delete mode 100644 streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/template/PipelineGenerator.java delete mode 100644 streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/template/PipelineTemplateGenerator.java delete mode 100644 streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/template/PipelineTemplateInvocationGenerator.java delete mode 100644 streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/template/PipelineTemplateInvocationHandler.java delete mode 100644 streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/template/PipelineTemplateManagement.java create mode 100644 streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/template/compact/CompactPipelineTemplateManagement.java create mode 100644 streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/template/compact/MatchingStreamFinder.java delete mode 100644 streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/template/instances/DataLakePipelineTemplate.java rename streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/template/instances/{PipelineTemplate.java => DefaultPipelineTemplateProvider.java} (73%) create mode 100644 streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/template/instances/PersistDataLakePipelineTemplate.java create mode 100644 streampipes-rest-shared/src/main/java/org/apache/streampipes/rest/shared/constants/SpMediaType.java delete mode 100644 streampipes-sdk/src/main/java/org/apache/streampipes/sdk/builder/BoundPipelineElementBuilder.java delete mode 100644 streampipes-sdk/src/main/java/org/apache/streampipes/sdk/builder/PipelineTemplateBuilder.java create mode 100644 streampipes-service-core/src/main/java/org/apache/streampipes/service/core/migrations/v970/AddDataLakePipelineTemplateMigration.java create mode 100644 ui/projects/streampipes/platform-services/src/lib/apis/compact-pipeline.service.ts create mode 100644 ui/src/app/connect/components/adapter-code-panel/adapter-code-panel.component.html create mode 100644 ui/src/app/connect/components/adapter-code-panel/adapter-code-panel.component.scss create mode 100644 ui/src/app/connect/components/adapter-code-panel/adapter-code-panel.component.ts create mode 100644 ui/src/app/connect/components/adapter-details/adapter-details-code/adapter-details-code.component.html create mode 100644 ui/src/app/connect/components/adapter-details/adapter-details-code/adapter-details-code.component.ts create mode 100644 ui/src/app/core-ui/configuration-code-panel/configuration-code-panel.component.html create mode 100644 ui/src/app/core-ui/configuration-code-panel/configuration-code-panel.component.scss create mode 100644 ui/src/app/core-ui/configuration-code-panel/configuration-code-panel.component.ts rename ui/src/app/{connect/filter => core-ui/pipes}/json-pretty-print.pipe.ts (100%) create mode 100644 ui/src/app/core-ui/pipes/yaml-pretty-print.pipe.ts create mode 100644 ui/src/app/editor/dialog/add-template-dialog/add-template-dialog.component.html create mode 100644 ui/src/app/editor/dialog/add-template-dialog/add-template-dialog.component.ts create mode 100644 ui/src/app/editor/dialog/add-template-dialog/template-selection/template-selection.component.html create mode 100644 ui/src/app/editor/dialog/add-template-dialog/template-selection/template-selection.component.scss create mode 100644 ui/src/app/editor/dialog/add-template-dialog/template-selection/template-selection.component.ts create mode 100644 ui/src/app/pipeline-details/dialogs/pipeline-code/pipeline-code-dialog.component.html create mode 100644 ui/src/app/pipeline-details/dialogs/pipeline-code/pipeline-code-dialog.component.ts diff --git a/streampipes-connect-management/src/main/java/org/apache/streampipes/connect/management/compact/PersistPipelineHandler.java b/streampipes-connect-management/src/main/java/org/apache/streampipes/connect/management/compact/PersistPipelineHandler.java index 7a262fca26..dd0afb7ef7 100644 --- a/streampipes-connect-management/src/main/java/org/apache/streampipes/connect/management/compact/PersistPipelineHandler.java +++ b/streampipes-connect-management/src/main/java/org/apache/streampipes/connect/management/compact/PersistPipelineHandler.java @@ -18,72 +18,93 @@ package org.apache.streampipes.connect.management.compact; -import org.apache.streampipes.manager.template.PipelineTemplateManagement; +import org.apache.streampipes.manager.pipeline.PipelineManager; +import org.apache.streampipes.manager.pipeline.compact.CompactPipelineManagement; import org.apache.streampipes.model.connect.adapter.AdapterDescription; +import org.apache.streampipes.model.connect.adapter.compact.CreateOptions; import org.apache.streampipes.model.pipeline.PipelineOperationStatus; +import org.apache.streampipes.model.pipeline.compact.CompactPipeline; +import org.apache.streampipes.model.pipeline.compact.CompactPipelineElement; import org.apache.streampipes.model.schema.EventProperty; import org.apache.streampipes.model.schema.EventPropertyPrimitive; -import org.apache.streampipes.model.staticproperty.FreeTextStaticProperty; -import org.apache.streampipes.model.staticproperty.MappingPropertyUnary; -import org.apache.streampipes.model.staticproperty.OneOfStaticProperty; -import org.apache.streampipes.model.template.PipelineTemplateInvocation; +import org.apache.streampipes.model.schema.PropertyScope; +import org.apache.streampipes.model.template.CompactPipelineTemplate; +import org.apache.streampipes.storage.api.CRUDStorage; import org.apache.streampipes.vocabulary.SO; -public class PersistPipelineHandler { +import java.util.List; +import java.util.Map; + +import static org.apache.streampipes.manager.template.instances.PersistDataLakePipelineTemplate.DATA_LAKE_CONNECTOR_ID; +import static org.apache.streampipes.manager.template.instances.PersistDataLakePipelineTemplate.DATA_LAKE_DIMENSIONS_FIELD; +import static org.apache.streampipes.manager.template.instances.PersistDataLakePipelineTemplate.DATA_LAKE_MEASUREMENT_FIELD; +import static org.apache.streampipes.manager.template.instances.PersistDataLakePipelineTemplate.DATA_LAKE_TEMPLATE_ID; +import static org.apache.streampipes.manager.template.instances.PersistDataLakePipelineTemplate.DATA_LAKE_TIMESTAMP_FIELD; - private static final String templateId = "org.apache.streampipes.manager.template.instances.DataLakePipelineTemplate"; - private static final String configPrefix = "jsplumb_domId2"; +public class PersistPipelineHandler { - private final PipelineTemplateManagement pipelineTemplateManagement; + private final CRUDStorage templateStorage; + private final CompactPipelineManagement pipelineManagement; private final String authenticatedUserSid; - public PersistPipelineHandler(PipelineTemplateManagement pipelineTemplateManagement, + public PersistPipelineHandler(CRUDStorage templateStorage, + CompactPipelineManagement pipelineManagement, String authenticatedUserSid) { - this.pipelineTemplateManagement = pipelineTemplateManagement; + this.templateStorage = templateStorage; + this.pipelineManagement = pipelineManagement; this.authenticatedUserSid = authenticatedUserSid; } - public PipelineOperationStatus createAndStartPersistPipeline(AdapterDescription adapterDescription) { - var pipelineTemplateInvocation = pipelineTemplateManagement.prepareInvocation( - adapterDescription.getCorrespondingDataStreamElementId(), - templateId - ); - - applyPipelineName(pipelineTemplateInvocation, adapterDescription.getName()); - applyDataLakeConfig(pipelineTemplateInvocation, adapterDescription); - - return pipelineTemplateManagement.createAndStartPipeline(pipelineTemplateInvocation, authenticatedUserSid); + public PipelineOperationStatus createAndStartPersistPipeline(AdapterDescription adapterDescription) throws Exception { + var template = getTemplate(); + if (template != null) { + var compactPipeline = new CompactPipeline( + String.format("persist-%s", adapterDescription.getName().replaceAll(" ", "-")), + String.format("Persist %s", adapterDescription.getName()), + null, + makeTemplateConfig(adapterDescription, template.getPipeline()), + new CreateOptions(false, true) + ); + var pipelineGenerationResult = pipelineManagement.makePipeline(compactPipeline); + if (pipelineGenerationResult.allPipelineElementsValid()) { + String pipelineId = PipelineManager.addPipeline(authenticatedUserSid, pipelineGenerationResult.pipeline()); + if (compactPipeline.createOptions().start()) { + return PipelineManager.startPipeline(pipelineId); + } + } + } + throw new IllegalArgumentException("Could not start persist pipeline"); } - private void applyPipelineName(PipelineTemplateInvocation pipelineTemplateInvocation, - String adapterName) { - pipelineTemplateInvocation.setPipelineTemplateId(templateId); - pipelineTemplateInvocation.setKviName(adapterName); + private CompactPipelineTemplate getTemplate() { + return this.templateStorage.getElementById(DATA_LAKE_TEMPLATE_ID); } - private void applyDataLakeConfig(PipelineTemplateInvocation pipelineTemplateInvocation, - AdapterDescription adapterDescription) { - pipelineTemplateInvocation.getStaticProperties().forEach(sp -> { - if (sp.getInternalName().equalsIgnoreCase(withPrefix("db_measurement"))) { - ((FreeTextStaticProperty) sp).setValue(adapterDescription.getName()); - } - if (sp.getInternalName().equalsIgnoreCase(withPrefix("timestamp_mapping"))) { - ((MappingPropertyUnary) sp).setSelectedProperty( - String.format("s0::%s", getTimestampField(adapterDescription) - )); - } - if (sp.getInternalName().equalsIgnoreCase(withPrefix("schema_update"))) { - ((OneOfStaticProperty) sp).getOptions().forEach(o -> { - if (o.getName().equals("Update schema")) { - o.setSelected(true); - } - }); - } - }); + private List makeTemplateConfig(AdapterDescription adapterDescription, + List pipelineElements) { + pipelineElements.get(0).configuration().addAll( + List.of( + Map.of(DATA_LAKE_MEASUREMENT_FIELD, adapterDescription.getName()), + Map.of(DATA_LAKE_TIMESTAMP_FIELD, String.format("s0::%s", getTimestampField(adapterDescription))), + Map.of(DATA_LAKE_DIMENSIONS_FIELD, getDimensions(adapterDescription)) + ) + ); + pipelineElements.add(new CompactPipelineElement( + "stream", + DATA_LAKE_CONNECTOR_ID, + adapterDescription.getCorrespondingDataStreamElementId(), + null, + null + )); + return pipelineElements; } - private String withPrefix(String config) { - return configPrefix + config; + private List getDimensions(AdapterDescription adapterDescription) { + return adapterDescription.getEventSchema().getEventProperties() + .stream() + .filter(ep -> ep.getPropertyScope().equalsIgnoreCase(PropertyScope.DIMENSION_PROPERTY.name())) + .map(EventProperty::getRuntimeName) + .toList(); } private String getTimestampField(AdapterDescription adapterDescription) { diff --git a/streampipes-connect-management/src/main/java/org/apache/streampipes/connect/management/compact/generator/AdapterEnrichmentRuleGenerator.java b/streampipes-connect-management/src/main/java/org/apache/streampipes/connect/management/compact/generator/AdapterEnrichmentRuleGenerator.java index 5abb7a99ce..9c19b30791 100644 --- a/streampipes-connect-management/src/main/java/org/apache/streampipes/connect/management/compact/generator/AdapterEnrichmentRuleGenerator.java +++ b/streampipes-connect-management/src/main/java/org/apache/streampipes/connect/management/compact/generator/AdapterEnrichmentRuleGenerator.java @@ -25,6 +25,8 @@ public class AdapterEnrichmentRuleGenerator implements AdapterModelGenerator { + public static final String TIMESTAMP_FIELD = "timestamp"; + @Override public void apply(AdapterDescription adapterDescription, CompactAdapter compactAdapter) throws Exception { @@ -33,7 +35,7 @@ public void apply(AdapterDescription adapterDescription, var timestampRule = new AddTimestampRuleDescription(compactAdapter.enrich().timestamp()); adapterDescription.getRules().add(timestampRule); adapterDescription.getEventSchema().addEventProperty( - EpProperties.timestampProperty("timestamp") + EpProperties.timestampProperty(TIMESTAMP_FIELD) ); } } diff --git a/streampipes-connect-management/src/main/java/org/apache/streampipes/connect/management/compact/generator/CompactAdapterGenerator.java b/streampipes-connect-management/src/main/java/org/apache/streampipes/connect/management/compact/generator/CompactAdapterGenerator.java new file mode 100644 index 0000000000..6857d0c4ef --- /dev/null +++ b/streampipes-connect-management/src/main/java/org/apache/streampipes/connect/management/compact/generator/CompactAdapterGenerator.java @@ -0,0 +1,123 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.streampipes.connect.management.compact.generator; + +import org.apache.streampipes.connect.shared.preprocessing.convert.ToOriginalSchemaConverter; +import org.apache.streampipes.manager.template.CompactConfigGenerator; +import org.apache.streampipes.model.connect.adapter.AdapterDescription; +import org.apache.streampipes.model.connect.adapter.compact.CompactEventProperty; +import org.apache.streampipes.model.connect.adapter.compact.CreateOptions; +import org.apache.streampipes.model.connect.adapter.compact.EnrichmentConfig; +import org.apache.streampipes.model.connect.adapter.compact.TransformationConfig; +import org.apache.streampipes.model.connect.rules.TransformationRuleDescription; +import org.apache.streampipes.model.connect.rules.schema.RenameRuleDescription; +import org.apache.streampipes.model.connect.rules.value.AddTimestampRuleDescription; +import org.apache.streampipes.model.connect.rules.value.UnitTransformRuleDescription; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public class CompactAdapterGenerator { + + private final AdapterDescription adapterDescription; + + public CompactAdapterGenerator(AdapterDescription adapterDescription) { + this.adapterDescription = adapterDescription; + } + + public List> getConfig() { + var configs = new ArrayList>(); + adapterDescription.getConfig().forEach(c -> { + configs.add(new CompactConfigGenerator(c).toTemplateValue()); + }); + return configs; + } + + public Map getSchema() { + var map = new HashMap(); + var originalProperties = new ToOriginalSchemaConverter( + adapterDescription.getEventSchema().getEventProperties() + ).getTransformedProperties(); + originalProperties + .forEach(ep -> map.put(ep.getRuntimeName(), new CompactEventProperty( + ep.getLabel(), + ep.getDescription(), + ep.getPropertyScope(), + ep.getSemanticType() + ))); + return map; + } + + public EnrichmentConfig getEnrichmentConfig() { + if (hasTimestampEnrichmentRule()) { + return new EnrichmentConfig(AdapterEnrichmentRuleGenerator.TIMESTAMP_FIELD); + } else { + return null; + } + } + + public TransformationConfig getTransformationConfig() { + var renameRules = new HashMap(); + var unitTransformRules = new HashMap(); + if (hasTransformationRule()) { + if (hasRule(RenameRuleDescription.class)) { + var rules = getRules(RenameRuleDescription.class); + rules.forEach(rule -> { + renameRules.put(rule.getOldRuntimeKey(), rule.getNewRuntimeKey()); + }); + } else if (hasRule(UnitTransformRuleDescription.class)) { + var rules = getRules(UnitTransformRuleDescription.class); + rules.forEach(rule -> { + unitTransformRules.put(rule.getRuntimeKey(), rule.getToUnitRessourceURL()); + }); + } + } + return new TransformationConfig(renameRules, unitTransformRules); + } + + public CreateOptions getCreateOptions() { + return new CreateOptions( + true, + true + ); + } + + private boolean hasTimestampEnrichmentRule() { + return hasRule(AddTimestampRuleDescription.class); + } + + private boolean hasTransformationRule() { + return adapterDescription.getRules().stream() + .anyMatch(r -> hasRule(RenameRuleDescription.class) || hasRule(UnitTransformRuleDescription.class)); + } + + private boolean hasRule(Class rule) { + return adapterDescription.getRules().stream().anyMatch(r -> r.getClass().equals(rule)); + } + + private List getRules(Class rule) { + return adapterDescription.getRules() + .stream() + .filter(rule::isInstance) + .map(rule::cast) + .toList(); + } +} diff --git a/streampipes-connect-management/src/main/java/org/apache/streampipes/connect/management/management/CompactAdapterManagement.java b/streampipes-connect-management/src/main/java/org/apache/streampipes/connect/management/management/CompactAdapterManagement.java index a973a22d85..12db048447 100644 --- a/streampipes-connect-management/src/main/java/org/apache/streampipes/connect/management/management/CompactAdapterManagement.java +++ b/streampipes-connect-management/src/main/java/org/apache/streampipes/connect/management/management/CompactAdapterManagement.java @@ -19,6 +19,7 @@ package org.apache.streampipes.connect.management.management; import org.apache.streampipes.connect.management.compact.generator.AdapterModelGenerator; +import org.apache.streampipes.connect.management.compact.generator.CompactAdapterGenerator; import org.apache.streampipes.model.connect.adapter.AdapterDescription; import org.apache.streampipes.model.connect.adapter.compact.CompactAdapter; import org.apache.streampipes.storage.api.IAdapterStorage; @@ -45,6 +46,22 @@ public AdapterDescription convertToAdapterDescription(CompactAdapter compactAdap return adapterDescription; } + public CompactAdapter convertToCompactAdapter(AdapterDescription adapterDescription) throws Exception { + var generator = new CompactAdapterGenerator(adapterDescription); + + return new CompactAdapter( + adapterDescription.getElementId(), + adapterDescription.getName(), + adapterDescription.getDescription(), + adapterDescription.getAppId(), + generator.getConfig(), + generator.getSchema(), + generator.getEnrichmentConfig(), + generator.getTransformationConfig(), + generator.getCreateOptions() + ); + } + public AdapterDescription convertToAdapterDescription(CompactAdapter compactAdapter, AdapterDescription existingAdapter) throws Exception { var adapterDescription = convertToAdapterDescription(compactAdapter); diff --git a/streampipes-model/src/main/java/org/apache/streampipes/model/connect/adapter/compact/CompactAdapter.java b/streampipes-model/src/main/java/org/apache/streampipes/model/connect/adapter/compact/CompactAdapter.java index d8a0a93f29..1129736376 100644 --- a/streampipes-model/src/main/java/org/apache/streampipes/model/connect/adapter/compact/CompactAdapter.java +++ b/streampipes-model/src/main/java/org/apache/streampipes/model/connect/adapter/compact/CompactAdapter.java @@ -36,9 +36,12 @@ package org.apache.streampipes.model.connect.adapter.compact; +import org.apache.streampipes.model.shared.annotation.TsModel; + import java.util.List; import java.util.Map; +@TsModel public record CompactAdapter( String id, String name, diff --git a/streampipes-model/src/main/java/org/apache/streampipes/model/connect/adapter/compact/CompactEventProperty.java b/streampipes-model/src/main/java/org/apache/streampipes/model/connect/adapter/compact/CompactEventProperty.java index c0cdd7e6af..479b42b0dc 100644 --- a/streampipes-model/src/main/java/org/apache/streampipes/model/connect/adapter/compact/CompactEventProperty.java +++ b/streampipes-model/src/main/java/org/apache/streampipes/model/connect/adapter/compact/CompactEventProperty.java @@ -18,6 +18,9 @@ package org.apache.streampipes.model.connect.adapter.compact; +import com.fasterxml.jackson.annotation.JsonInclude; + +@JsonInclude(JsonInclude.Include.NON_NULL) public record CompactEventProperty( String label, String description, diff --git a/streampipes-model/src/main/java/org/apache/streampipes/model/connect/adapter/compact/CreateOptions.java b/streampipes-model/src/main/java/org/apache/streampipes/model/connect/adapter/compact/CreateOptions.java index 6fc266ff21..ee8ea06f61 100644 --- a/streampipes-model/src/main/java/org/apache/streampipes/model/connect/adapter/compact/CreateOptions.java +++ b/streampipes-model/src/main/java/org/apache/streampipes/model/connect/adapter/compact/CreateOptions.java @@ -18,6 +18,9 @@ package org.apache.streampipes.model.connect.adapter.compact; -public record CreateOptions(boolean persist, - boolean start) { +import com.fasterxml.jackson.annotation.JsonInclude; + +@JsonInclude(JsonInclude.Include.NON_NULL) +public record CreateOptions(Boolean persist, + Boolean start) { } diff --git a/streampipes-model/src/main/java/org/apache/streampipes/model/pipeline/compact/CompactPipeline.java b/streampipes-model/src/main/java/org/apache/streampipes/model/pipeline/compact/CompactPipeline.java index 59948aeb03..006a9cda12 100644 --- a/streampipes-model/src/main/java/org/apache/streampipes/model/pipeline/compact/CompactPipeline.java +++ b/streampipes-model/src/main/java/org/apache/streampipes/model/pipeline/compact/CompactPipeline.java @@ -19,9 +19,11 @@ package org.apache.streampipes.model.pipeline.compact; import org.apache.streampipes.model.connect.adapter.compact.CreateOptions; +import org.apache.streampipes.model.shared.annotation.TsModel; import java.util.List; +@TsModel public record CompactPipeline( String id, String name, diff --git a/streampipes-model/src/main/java/org/apache/streampipes/model/pipeline/compact/CompactPipelineElement.java b/streampipes-model/src/main/java/org/apache/streampipes/model/pipeline/compact/CompactPipelineElement.java index 0f976c8145..b145c2aad3 100644 --- a/streampipes-model/src/main/java/org/apache/streampipes/model/pipeline/compact/CompactPipelineElement.java +++ b/streampipes-model/src/main/java/org/apache/streampipes/model/pipeline/compact/CompactPipelineElement.java @@ -18,9 +18,12 @@ package org.apache.streampipes.model.pipeline.compact; +import com.fasterxml.jackson.annotation.JsonInclude; + import java.util.List; import java.util.Map; +@JsonInclude(JsonInclude.Include.NON_NULL) public record CompactPipelineElement(String type, String ref, String id, diff --git a/streampipes-model/src/main/java/org/apache/streampipes/model/template/BoundPipelineElement.java b/streampipes-model/src/main/java/org/apache/streampipes/model/template/BoundPipelineElement.java deleted file mode 100644 index 267b3511be..0000000000 --- a/streampipes-model/src/main/java/org/apache/streampipes/model/template/BoundPipelineElement.java +++ /dev/null @@ -1,57 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ -package org.apache.streampipes.model.template; - -import org.apache.streampipes.model.base.InvocableStreamPipesEntity; -import org.apache.streampipes.model.util.Cloner; - -import java.util.ArrayList; -import java.util.List; - -public class BoundPipelineElement { - - private InvocableStreamPipesEntity pipelineElementTemplate; - - private List connectedTo; - - public BoundPipelineElement() { - super(); - this.connectedTo = new ArrayList<>(); - } - - public BoundPipelineElement(BoundPipelineElement other) { - this.pipelineElementTemplate = other.getPipelineElementTemplate(); - this.connectedTo = new Cloner().boundPipelineElements(other.getConnectedTo()); - } - - public InvocableStreamPipesEntity getPipelineElementTemplate() { - return pipelineElementTemplate; - } - - public void setPipelineElementTemplate(InvocableStreamPipesEntity pipelineElementTemplate) { - this.pipelineElementTemplate = pipelineElementTemplate; - } - - public List getConnectedTo() { - return connectedTo; - } - - public void setConnectedTo(List connectedTo) { - this.connectedTo = connectedTo; - } -} diff --git a/streampipes-model/src/main/java/org/apache/streampipes/model/template/CompactPipelineTemplate.java b/streampipes-model/src/main/java/org/apache/streampipes/model/template/CompactPipelineTemplate.java new file mode 100644 index 0000000000..6b662c33bc --- /dev/null +++ b/streampipes-model/src/main/java/org/apache/streampipes/model/template/CompactPipelineTemplate.java @@ -0,0 +1,106 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.streampipes.model.template; + +import org.apache.streampipes.model.pipeline.compact.CompactPipeline; +import org.apache.streampipes.model.pipeline.compact.CompactPipelineElement; +import org.apache.streampipes.model.shared.annotation.TsModel; +import org.apache.streampipes.model.shared.api.Storable; + +import com.fasterxml.jackson.annotation.JsonAlias; +import com.google.gson.annotations.SerializedName; + +import java.util.List; + +@TsModel +public class CompactPipelineTemplate implements Storable { + + @JsonAlias("id") + protected @SerializedName("_id") String elementId; + + @JsonAlias("_rev") + protected @SerializedName("_rev") String rev; + + private String name; + private String description; + private List pipeline; + private PipelinePlaceholders placeholders; + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getDescription() { + return description; + } + + public void setDescription(String description) { + this.description = description; + } + + public List getPipeline() { + return pipeline; + } + + public void setPipeline(List pipeline) { + this.pipeline = pipeline; + } + + public PipelinePlaceholders getPlaceholders() { + return placeholders; + } + + public void setPlaceholders(PipelinePlaceholders placeholders) { + this.placeholders = placeholders; + } + + @Override + public String getRev() { + return rev; + } + + @Override + public void setRev(String rev) { + this.rev = rev; + } + + @Override + public String getElementId() { + return elementId; + } + + @Override + public void setElementId(String elementId) { + this.elementId = elementId; + } + + public CompactPipeline toCompactPipeline() { + return new CompactPipeline( + null, + null, + null, + this.getPipeline(), + null + ); + } +} diff --git a/streampipes-model/src/main/java/org/apache/streampipes/model/template/PipelineElementTemplateConfig.java b/streampipes-model/src/main/java/org/apache/streampipes/model/template/PipelineElementTemplateConfig.java deleted file mode 100644 index 0d52913e46..0000000000 --- a/streampipes-model/src/main/java/org/apache/streampipes/model/template/PipelineElementTemplateConfig.java +++ /dev/null @@ -1,67 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ -package org.apache.streampipes.model.template; - -public class PipelineElementTemplateConfig { - - private boolean editable; - private boolean displayed; - - private Object value; - - public PipelineElementTemplateConfig(boolean editable, boolean displayed, Object value) { - this.editable = editable; - this.displayed = displayed; - this.value = value; - } - - public PipelineElementTemplateConfig() { - } - - private PipelineElementTemplateConfig(Object value) { - this.value = value; - } - - public static PipelineElementTemplateConfig from(Object value) { - return new PipelineElementTemplateConfig(value); - } - - public boolean isEditable() { - return editable; - } - - public void setEditable(boolean editable) { - this.editable = editable; - } - - public boolean isDisplayed() { - return displayed; - } - - public void setDisplayed(boolean displayed) { - this.displayed = displayed; - } - - public Object getValue() { - return value; - } - - public void setValue(Object value) { - this.value = value; - } -} diff --git a/streampipes-model/src/main/java/org/apache/streampipes/model/template/PipelinePlaceholderConfig.java b/streampipes-model/src/main/java/org/apache/streampipes/model/template/PipelinePlaceholderConfig.java new file mode 100644 index 0000000000..b1fea0d342 --- /dev/null +++ b/streampipes-model/src/main/java/org/apache/streampipes/model/template/PipelinePlaceholderConfig.java @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.streampipes.model.template; + +public record PipelinePlaceholderConfig(String ref, + String id) { +} diff --git a/streampipes-model/src/main/java/org/apache/streampipes/model/template/PipelineTemplateDescriptionContainer.java b/streampipes-model/src/main/java/org/apache/streampipes/model/template/PipelinePlaceholders.java similarity index 61% rename from streampipes-model/src/main/java/org/apache/streampipes/model/template/PipelineTemplateDescriptionContainer.java rename to streampipes-model/src/main/java/org/apache/streampipes/model/template/PipelinePlaceholders.java index e523fa0bb6..9f247f312b 100644 --- a/streampipes-model/src/main/java/org/apache/streampipes/model/template/PipelineTemplateDescriptionContainer.java +++ b/streampipes-model/src/main/java/org/apache/streampipes/model/template/PipelinePlaceholders.java @@ -15,30 +15,11 @@ * limitations under the License. * */ + package org.apache.streampipes.model.template; -import java.util.ArrayList; import java.util.List; -public class PipelineTemplateDescriptionContainer { - - private List list; - - public PipelineTemplateDescriptionContainer() { - super(); - this.list = new ArrayList<>(); - } - - public PipelineTemplateDescriptionContainer(List dataStreams) { - super(); - this.list = dataStreams; - } - - public List getList() { - return list; - } - - public void setList(List list) { - this.list = list; - } +public record PipelinePlaceholders(List requiredStreamInputs, + List requiredConfigs) { } diff --git a/streampipes-model/src/main/java/org/apache/streampipes/model/template/PipelineTemplateDescription.java b/streampipes-model/src/main/java/org/apache/streampipes/model/template/PipelineTemplateDescription.java deleted file mode 100644 index 5a00f6c754..0000000000 --- a/streampipes-model/src/main/java/org/apache/streampipes/model/template/PipelineTemplateDescription.java +++ /dev/null @@ -1,87 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ -package org.apache.streampipes.model.template; - -import org.apache.streampipes.model.SpDataStream; -import org.apache.streampipes.model.base.NamedStreamPipesEntity; -import org.apache.streampipes.model.shared.annotation.TsModel; -import org.apache.streampipes.model.util.Cloner; - -import java.util.ArrayList; -import java.util.List; - -@TsModel -public class PipelineTemplateDescription extends NamedStreamPipesEntity { - - private List boundTo; - - public PipelineTemplateDescription() { - super(); - this.boundTo = new ArrayList<>(); - } - - public PipelineTemplateDescription(PipelineTemplateDescription other) { - super(other); - // TODO use cloner - if (other.getBoundTo() != null) { - this.boundTo = new Cloner().boundPipelineElements(other.getBoundTo()); - } - //this.pipelineTemplateName = other.getPipelineTemplateName(); - //this.pipelineTemplateDescription = other.getPipelineTemplateDescription(); - //this.pipelineTemplateId = other.getPipelineTemplateId(); - } - - public PipelineTemplateDescription(String elementName, SpDataStream requiredStream, - List connectedTo) { - super(elementName); - this.boundTo = connectedTo; - } - - public List getBoundTo() { - return boundTo; - } - - public void setBoundTo(List boundTo) { - this.boundTo = boundTo; - } - - public String getPipelineTemplateName() { - return super.getName(); - } - - public void setPipelineTemplateName(String pipelineTemplateName) { - super.setName(pipelineTemplateName); - } - - public String getPipelineTemplateDescription() { - return super.getDescription(); - } - - public void setPipelineTemplateDescription(String pipelineTemplateDescription) { - super.setDescription(pipelineTemplateDescription); - } - - public String getPipelineTemplateId() { - return super.getElementId(); - } - - public void setPipelineTemplateId(String pipelineTemplateId) { - super.setElementId(pipelineTemplateId); - super.setAppId(pipelineTemplateId); - } -} diff --git a/streampipes-model/src/main/java/org/apache/streampipes/model/template/PipelineTemplateGenerationRequest.java b/streampipes-model/src/main/java/org/apache/streampipes/model/template/PipelineTemplateGenerationRequest.java new file mode 100644 index 0000000000..b25b1caf27 --- /dev/null +++ b/streampipes-model/src/main/java/org/apache/streampipes/model/template/PipelineTemplateGenerationRequest.java @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.streampipes.model.template; + +import org.apache.streampipes.model.shared.annotation.TsModel; + +import java.util.Map; + +@TsModel +public record PipelineTemplateGenerationRequest(CompactPipelineTemplate template, + Map streams, + String pipelineName, + String pipelineDescription) { +} diff --git a/streampipes-model/src/main/java/org/apache/streampipes/model/template/PipelineTemplateInvocation.java b/streampipes-model/src/main/java/org/apache/streampipes/model/template/PipelineTemplateInvocation.java deleted file mode 100644 index e36de80bcd..0000000000 --- a/streampipes-model/src/main/java/org/apache/streampipes/model/template/PipelineTemplateInvocation.java +++ /dev/null @@ -1,97 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ -package org.apache.streampipes.model.template; - -import org.apache.streampipes.model.shared.annotation.TsModel; -import org.apache.streampipes.model.staticproperty.StaticProperty; -import org.apache.streampipes.model.util.Cloner; - -import com.fasterxml.jackson.annotation.JsonTypeInfo; - -import java.util.ArrayList; -import java.util.List; - -@JsonTypeInfo(use = JsonTypeInfo.Id.CLASS, property = "@class") -@TsModel -public class PipelineTemplateInvocation { - - private String kviName; - - private String dataStreamId; - - private String pipelineTemplateId; - - private PipelineTemplateDescription pipelineTemplateDescription; - - private List staticProperties; - - public PipelineTemplateInvocation() { - super(); - this.staticProperties = new ArrayList<>(); - } - - public PipelineTemplateInvocation(PipelineTemplateInvocation other) { - this.kviName = other.getKviName(); - this.dataStreamId = other.getDataStreamId(); - this.pipelineTemplateId = other.getPipelineTemplateId(); - - if (other.getStaticProperties() != null) { - this.staticProperties = new Cloner().staticProperties(other.getStaticProperties()); - } - } - - public String getKviName() { - return kviName; - } - - public void setKviName(String kviName) { - this.kviName = kviName; - } - - public String getDataStreamId() { - return dataStreamId; - } - - public void setDataStreamId(String dataStreamId) { - this.dataStreamId = dataStreamId; - } - - public List getStaticProperties() { - return staticProperties; - } - - public void setStaticProperties(List staticProperties) { - this.staticProperties = staticProperties; - } - - public PipelineTemplateDescription getPipelineTemplateDescription() { - return pipelineTemplateDescription; - } - - public void setPipelineTemplateDescription(PipelineTemplateDescription pipelineTemplateDescription) { - this.pipelineTemplateDescription = pipelineTemplateDescription; - } - - public String getPipelineTemplateId() { - return pipelineTemplateId; - } - - public void setPipelineTemplateId(String pipelineTemplateId) { - this.pipelineTemplateId = pipelineTemplateId; - } -} diff --git a/streampipes-model/src/main/java/org/apache/streampipes/model/util/Cloner.java b/streampipes-model/src/main/java/org/apache/streampipes/model/util/Cloner.java index 853415270c..bce767d46c 100644 --- a/streampipes-model/src/main/java/org/apache/streampipes/model/util/Cloner.java +++ b/streampipes-model/src/main/java/org/apache/streampipes/model/util/Cloner.java @@ -73,7 +73,6 @@ import org.apache.streampipes.model.staticproperty.StaticPropertyAlternatives; import org.apache.streampipes.model.staticproperty.StaticPropertyGroup; import org.apache.streampipes.model.staticproperty.SupportedProperty; -import org.apache.streampipes.model.template.BoundPipelineElement; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -254,13 +253,6 @@ public TopicDefinition topicDefinition(TopicDefinition topicDefinition) { } } - public List boundPipelineElements(List boundPipelineElements) { - return boundPipelineElements - .stream() - .map(BoundPipelineElement::new) - .collect(Collectors.toList()); - } - public List cloneDescriptions(List pipelineElementDescriptions) { return pipelineElementDescriptions .stream() diff --git a/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/pipeline/compact/CompactPipelineManagement.java b/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/pipeline/compact/CompactPipelineManagement.java index 1a0a1cd44e..e0137e13b4 100644 --- a/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/pipeline/compact/CompactPipelineManagement.java +++ b/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/pipeline/compact/CompactPipelineManagement.java @@ -19,12 +19,16 @@ package org.apache.streampipes.manager.pipeline.compact; import org.apache.streampipes.manager.matching.PipelineVerificationHandlerV2; +import org.apache.streampipes.manager.pipeline.compact.generation.CompactPipelineConverter; import org.apache.streampipes.manager.pipeline.compact.generation.PipelineElementConfigurationStep; +import org.apache.streampipes.model.connect.adapter.compact.CreateOptions; import org.apache.streampipes.model.pipeline.Pipeline; import org.apache.streampipes.model.pipeline.PipelineModificationResult; import org.apache.streampipes.model.pipeline.compact.CompactPipeline; import org.apache.streampipes.storage.api.IPipelineElementDescriptionStorage; +import java.util.UUID; + public class CompactPipelineManagement { private final IPipelineElementDescriptionStorage storage; @@ -42,6 +46,16 @@ public PipelineModificationResult makePipeline(CompactPipeline compactPipeline) return new PipelineVerificationHandlerV2(pipeline).makeModifiedPipeline(); } + public CompactPipeline convertPipeline(Pipeline pipeline) { + return new CompactPipeline( + pipeline.getElementId() != null ? pipeline.getElementId() : UUID.randomUUID().toString(), + pipeline.getName(), + pipeline.getDescription(), + new CompactPipelineConverter().convert(pipeline), + new CreateOptions(null, false) + ); + } + private void applyPipelineBasics(CompactPipeline compactPipeline, Pipeline pipeline) { pipeline.setElementId(compactPipeline.id()); diff --git a/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/pipeline/compact/generation/CompactPipelineConverter.java b/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/pipeline/compact/generation/CompactPipelineConverter.java new file mode 100644 index 0000000000..bebc3b9286 --- /dev/null +++ b/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/pipeline/compact/generation/CompactPipelineConverter.java @@ -0,0 +1,85 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.streampipes.manager.pipeline.compact.generation; + +import org.apache.streampipes.manager.template.CompactConfigGenerator; +import org.apache.streampipes.model.pipeline.Pipeline; +import org.apache.streampipes.model.pipeline.compact.CompactPipelineElement; +import org.apache.streampipes.model.staticproperty.StaticProperty; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +public class CompactPipelineConverter { + + public List convert(Pipeline pipeline) { + var pipelineElements = new ArrayList(); + + pipeline.getStreams().stream() + .map(stream -> createElement( + PipelineElementConfigurationStep.STREAM_TYPE, + stream.getDom(), + stream.getElementId(), + null, + null)) + .forEach(pipelineElements::add); + + pipeline.getSepas().stream() + .map(processor -> createElement( + PipelineElementConfigurationStep.PROCESSOR_TYPE, + processor.getDom(), + processor.getAppId(), + processor.getConnectedTo(), + getConfig(processor.getStaticProperties()))) + .forEach(pipelineElements::add); + + pipeline.getActions().stream() + .map(sink -> createElement( + PipelineElementConfigurationStep.SINK_TYPE, + sink.getDom(), + sink.getAppId(), + sink.getConnectedTo(), + getConfig(sink.getStaticProperties()))) + .forEach(pipelineElements::add); + + return pipelineElements; + } + + private CompactPipelineElement createElement(String type, + String ref, + String elementId, + List connectedTo, + List> config) { + var connections = connectedTo != null ? connectedTo.stream() + .map(this::replaceId) + .toList() : null; + return new CompactPipelineElement(type, replaceId(ref), elementId, connections, config); + } + + public List> getConfig(List staticProperties) { + var configs = new ArrayList>(); + staticProperties.forEach(c -> configs.add(new CompactConfigGenerator(c).toTemplateValue())); + return configs; + } + + private String replaceId(String id) { + return id.replaceAll(InvocablePipelineElementGenerator.ID_PREFIX, ""); + } +} diff --git a/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/pipeline/compact/generation/InvocablePipelineElementGenerator.java b/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/pipeline/compact/generation/InvocablePipelineElementGenerator.java index 0f8d135830..8686a4b0a6 100644 --- a/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/pipeline/compact/generation/InvocablePipelineElementGenerator.java +++ b/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/pipeline/compact/generation/InvocablePipelineElementGenerator.java @@ -27,7 +27,7 @@ public class InvocablePipelineElementGenerator { - private static final String ID_PREFIX = "jsplumb_"; + public static final String ID_PREFIX = "jsplumb_"; public void apply(T element, CompactPipelineElement compatPipelineElement) { diff --git a/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/pipeline/compact/generation/PipelineElementConfigurationStep.java b/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/pipeline/compact/generation/PipelineElementConfigurationStep.java index ec5539414c..ec61f414be 100644 --- a/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/pipeline/compact/generation/PipelineElementConfigurationStep.java +++ b/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/pipeline/compact/generation/PipelineElementConfigurationStep.java @@ -28,9 +28,9 @@ public class PipelineElementConfigurationStep implements CompactPipelineGenerator { - private static final String StreamType = "stream"; - private static final String ProcessorType = "processor"; - private static final String SinkType = "sink"; + public static final String STREAM_TYPE = "stream"; + public static final String PROCESSOR_TYPE = "processor"; + public static final String SINK_TYPE = "sink"; private final IPipelineElementDescriptionStorage storage; @@ -42,11 +42,11 @@ public PipelineElementConfigurationStep(IPipelineElementDescriptionStorage stora public void apply(Pipeline pipeline, CompactPipeline compactPipeline) throws Exception { compactPipeline.pipelineElements().forEach(pe -> { - if (pe.type().equalsIgnoreCase(StreamType)) { + if (pe.type().equalsIgnoreCase(STREAM_TYPE)) { pipeline.getStreams().add(makeStream(pe)); - } else if (pe.type().equalsIgnoreCase(ProcessorType)) { + } else if (pe.type().equalsIgnoreCase(PROCESSOR_TYPE)) { pipeline.getSepas().add(makeProcessor(pe)); - } else if (pe.type().equalsIgnoreCase(SinkType)) { + } else if (pe.type().equalsIgnoreCase(SINK_TYPE)) { pipeline.getActions().add(makeSink(pe)); } }); diff --git a/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/setup/CouchDbInstallationStep.java b/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/setup/CouchDbInstallationStep.java index b010dbae56..538f8243ae 100644 --- a/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/setup/CouchDbInstallationStep.java +++ b/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/setup/CouchDbInstallationStep.java @@ -19,9 +19,11 @@ package org.apache.streampipes.manager.setup; import org.apache.streampipes.manager.setup.design.UserDesignDocument; +import org.apache.streampipes.manager.setup.tasks.AddDefaultPipelineTemplatesTask; import org.apache.streampipes.manager.setup.tasks.CreateAssetLinkTypeTask; import org.apache.streampipes.manager.setup.tasks.CreateDefaultAssetTask; import org.apache.streampipes.storage.couchdb.utils.Utils; +import org.apache.streampipes.storage.management.StorageDispatcher; import org.lightcouch.DesignDocument; import org.lightcouch.DesignDocument.MapReduce; @@ -48,6 +50,9 @@ public void install() { createViews(); new CreateAssetLinkTypeTask().execute(); new CreateDefaultAssetTask().execute(); + new AddDefaultPipelineTemplatesTask( + StorageDispatcher.INSTANCE.getNoSqlStore().getPipelineTemplateStorage() + ).execute(); } @Override diff --git a/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/setup/tasks/AddDefaultPipelineTemplatesTask.java b/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/setup/tasks/AddDefaultPipelineTemplatesTask.java new file mode 100644 index 0000000000..23ed1efbce --- /dev/null +++ b/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/setup/tasks/AddDefaultPipelineTemplatesTask.java @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.streampipes.manager.setup.tasks; + +import org.apache.streampipes.manager.template.instances.PersistDataLakePipelineTemplate; +import org.apache.streampipes.model.template.CompactPipelineTemplate; +import org.apache.streampipes.storage.api.CRUDStorage; + +public class AddDefaultPipelineTemplatesTask implements InstallationTask { + + CRUDStorage storage; + + public AddDefaultPipelineTemplatesTask(CRUDStorage storage) { + this.storage = storage; + } + + @Override + public void execute() { + storage.persist(new PersistDataLakePipelineTemplate().getTemplate()); + } +} diff --git a/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/template/CompactConfigGenerator.java b/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/template/CompactConfigGenerator.java new file mode 100644 index 0000000000..3ec54266b7 --- /dev/null +++ b/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/template/CompactConfigGenerator.java @@ -0,0 +1,211 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.streampipes.manager.template; + +import org.apache.streampipes.model.staticproperty.AnyStaticProperty; +import org.apache.streampipes.model.staticproperty.CodeInputStaticProperty; +import org.apache.streampipes.model.staticproperty.CollectionStaticProperty; +import org.apache.streampipes.model.staticproperty.ColorPickerStaticProperty; +import org.apache.streampipes.model.staticproperty.FileStaticProperty; +import org.apache.streampipes.model.staticproperty.FreeTextStaticProperty; +import org.apache.streampipes.model.staticproperty.MappingPropertyNary; +import org.apache.streampipes.model.staticproperty.MappingPropertyUnary; +import org.apache.streampipes.model.staticproperty.MatchingStaticProperty; +import org.apache.streampipes.model.staticproperty.OneOfStaticProperty; +import org.apache.streampipes.model.staticproperty.Option; +import org.apache.streampipes.model.staticproperty.RuntimeResolvableGroupStaticProperty; +import org.apache.streampipes.model.staticproperty.RuntimeResolvableTreeInputStaticProperty; +import org.apache.streampipes.model.staticproperty.SecretStaticProperty; +import org.apache.streampipes.model.staticproperty.SlideToggleStaticProperty; +import org.apache.streampipes.model.staticproperty.StaticProperty; +import org.apache.streampipes.model.staticproperty.StaticPropertyAlternative; +import org.apache.streampipes.model.staticproperty.StaticPropertyAlternatives; +import org.apache.streampipes.model.staticproperty.StaticPropertyGroup; +import org.apache.streampipes.model.staticproperty.StaticPropertyVisitor; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public class CompactConfigGenerator implements StaticPropertyVisitor { + + private final Map config; + private final StaticProperty staticProperty; + + public CompactConfigGenerator(StaticProperty staticProperty) { + this.config = new HashMap<>(); + this.staticProperty = staticProperty; + } + + public Map toTemplateValue() { + staticProperty.accept(this); + return config; + } + + @Override + public void visit(AnyStaticProperty property) { + addConfig( + property, + property.getOptions().stream().filter(Option::isSelected).map(Option::getName).toList() + ); + } + + @Override + public void visit(CodeInputStaticProperty codeInputStaticProperty) { + addConfig( + codeInputStaticProperty, + codeInputStaticProperty.getValue() + ); + } + + @Override + public void visit(CollectionStaticProperty collectionStaticProperty) { + config.put( + collectionStaticProperty.getInternalName(), + addListEntry(collectionStaticProperty.getMembers()) + ); + } + + @Override + public void visit(ColorPickerStaticProperty colorPickerStaticProperty) { + addConfig( + colorPickerStaticProperty, + colorPickerStaticProperty.getSelectedColor() + ); + } + + @Override + public void visit(FileStaticProperty fileStaticProperty) { + addConfig( + fileStaticProperty, + fileStaticProperty.getLocationPath() + ); + } + + @Override + public void visit(FreeTextStaticProperty freeTextStaticProperty) { + addConfig( + freeTextStaticProperty, + freeTextStaticProperty.getValue() + ); + } + + @Override + public void visit(MappingPropertyNary mappingPropertyNary) { + addConfig( + mappingPropertyNary, + mappingPropertyNary.getSelectedProperties() + ); + } + + @Override + public void visit(MappingPropertyUnary mappingPropertyUnary) { + addConfig( + mappingPropertyUnary, + mappingPropertyUnary.getSelectedProperty() + ); + } + + @Override + public void visit(MatchingStaticProperty matchingStaticProperty) { + // not supported + } + + @Override + public void visit(OneOfStaticProperty oneOfStaticProperty) { + addConfig( + oneOfStaticProperty, + oneOfStaticProperty.getOptions().stream().filter(Option::isSelected).findFirst().map(Option::getName) + ); + } + + @Override + public void visit(SecretStaticProperty secretStaticProperty) { + config.put(secretStaticProperty.getInternalName(), secretStaticProperty.getValue()); + config.put("encrypted", secretStaticProperty.getEncrypted()); + } + + @Override + public void visit(StaticPropertyAlternative staticPropertyAlternative) { + + } + + @Override + public void visit(StaticPropertyAlternatives staticPropertyAlternatives) { + var selectedAlternativeOpt = staticPropertyAlternatives.getAlternatives() + .stream() + .filter(StaticPropertyAlternative::getSelected) + .findFirst(); + if (selectedAlternativeOpt.isPresent()) { + var selectedAlternative = selectedAlternativeOpt.get(); + config.put(staticPropertyAlternatives.getInternalName(), selectedAlternative.getInternalName()); + if (selectedAlternative.getStaticProperty() != null) { + var alternative = new CompactConfigGenerator(selectedAlternative.getStaticProperty()).toTemplateValue(); + config.putAll(alternative); + } + } + } + + @Override + public void visit(StaticPropertyGroup staticPropertyGroup) { + config.putAll(addNestedEntry(staticPropertyGroup.getStaticProperties())); + } + + @Override + public void visit(SlideToggleStaticProperty slideToggleStaticProperty) { + addConfig( + slideToggleStaticProperty, + slideToggleStaticProperty.isSelected() + ); + } + + @Override + public void visit(RuntimeResolvableTreeInputStaticProperty treeInputStaticProperty) { + addConfig( + treeInputStaticProperty, + treeInputStaticProperty.getSelectedNodesInternalNames() + ); + } + + @Override + public void visit(RuntimeResolvableGroupStaticProperty groupStaticProperty) { + + } + + private void addConfig(StaticProperty staticProperty, Object value) { + config.put(staticProperty.getInternalName(), value); + } + + public List> addListEntry(List staticProperties) { + return staticProperties.stream() + .map(sp -> new CompactConfigGenerator(sp).toTemplateValue()) + .toList(); + } + + public Map addNestedEntry(List staticProperties) { + Map entry = new HashMap<>(); + + staticProperties.forEach(sp -> { + Map groupEntries = new CompactConfigGenerator(sp).toTemplateValue(); + entry.putAll(groupEntries); + }); + + return entry; + } +} diff --git a/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/template/PipelineElementTemplateVisitor.java b/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/template/PipelineElementTemplateVisitor.java index 11ec281927..7e91b9ce3e 100644 --- a/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/template/PipelineElementTemplateVisitor.java +++ b/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/template/PipelineElementTemplateVisitor.java @@ -27,7 +27,9 @@ import org.apache.streampipes.model.staticproperty.MappingPropertyUnary; import org.apache.streampipes.model.staticproperty.MatchingStaticProperty; import org.apache.streampipes.model.staticproperty.OneOfStaticProperty; +import org.apache.streampipes.model.staticproperty.Option; import org.apache.streampipes.model.staticproperty.RuntimeResolvableGroupStaticProperty; +import org.apache.streampipes.model.staticproperty.RuntimeResolvableOneOfStaticProperty; import org.apache.streampipes.model.staticproperty.RuntimeResolvableTreeInputStaticProperty; import org.apache.streampipes.model.staticproperty.SecretStaticProperty; import org.apache.streampipes.model.staticproperty.SlideToggleStaticProperty; @@ -132,6 +134,10 @@ public void visit(OneOfStaticProperty oneOfStaticProperty) { String value = getConfigValueAsString(oneOfStaticProperty); oneOfStaticProperty.getOptions().forEach(option -> option.setSelected(option.getName().equals(value))); + if (oneOfStaticProperty instanceof RuntimeResolvableOneOfStaticProperty + && oneOfStaticProperty.getOptions().isEmpty()) { + oneOfStaticProperty.setOptions(List.of(new Option(value, true))); + } } } @@ -203,8 +209,12 @@ public void visit(RuntimeResolvableTreeInputStaticProperty property) { } @Override - public void visit(RuntimeResolvableGroupStaticProperty groupStaticProperty) { - // TODO not yet supported + public void visit(RuntimeResolvableGroupStaticProperty staticPropertyGroup) { + staticPropertyGroup.getStaticProperties().forEach(group -> { + PipelineElementTemplateVisitor visitor = + new PipelineElementTemplateVisitor(configs); + group.accept(visitor); + }); } @@ -220,10 +230,6 @@ private Map getConfig(String key) { .orElseThrow(() -> new IllegalArgumentException(String.format("No key found: %s", key))); } -// private List> getConfigAsList(StaticProperty sp) { -// return getConfig(sp). -// } - private boolean hasKeyCaseInsensitive(String internalName, Map templateConfig) { return templateConfig @@ -289,29 +295,4 @@ private List> getCaseInsensitiveList(Map map } throw new IllegalArgumentException("Key '" + key + "' not found"); } - - -// private String getAsString(StaticProperty sp) { -// return configs.get(sp.getInternalName()).toString(); -// } -// -// private boolean getAsBoolean(StaticProperty sp) { -// return Boolean.parseBoolean(configs.get(sp.getInternalName()).toString()); -// } -// -// private Map getAsMap(StaticProperty sp) { -// return (Map) configs.get(sp.getInternalName()); -// } -// -// private Map getAsMap(StaticProperty sp, String subkey) { -// return (Map) getAsMap(sp).get(subkey); -// } -// -// private Map getAsMap(Map map, String key) { -// return (Map) map.get(key); -// } -// -// private List> getAsList(StaticProperty sp, String key) { -// return (List>) getAsMap(sp).get(key); -// } } diff --git a/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/template/PipelineGenerator.java b/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/template/PipelineGenerator.java deleted file mode 100644 index 134e37a1d2..0000000000 --- a/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/template/PipelineGenerator.java +++ /dev/null @@ -1,139 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ -package org.apache.streampipes.manager.template; - -import org.apache.streampipes.manager.matching.PipelineVerificationHandlerV2; -import org.apache.streampipes.model.SpDataStream; -import org.apache.streampipes.model.base.InvocableStreamPipesEntity; -import org.apache.streampipes.model.graph.DataProcessorInvocation; -import org.apache.streampipes.model.graph.DataSinkInvocation; -import org.apache.streampipes.model.message.PipelineModificationMessage; -import org.apache.streampipes.model.pipeline.Pipeline; -import org.apache.streampipes.model.pipeline.PipelineModification; -import org.apache.streampipes.model.template.BoundPipelineElement; -import org.apache.streampipes.model.template.PipelineTemplateDescription; -import org.apache.streampipes.storage.management.StorageDispatcher; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.UUID; - -public class PipelineGenerator { - - private final PipelineTemplateDescription pipelineTemplateDescription; - private final String datasetId; - private final Pipeline pipeline; - private final String pipelineName; - - private int count = 0; - - public PipelineGenerator(String datasetId, - PipelineTemplateDescription pipelineTemplateDescription, - String pipelineName) { - this.pipelineTemplateDescription = pipelineTemplateDescription; - this.datasetId = datasetId; - this.pipelineName = pipelineName; - this.pipeline = new Pipeline(); - } - - public Pipeline makePipeline() { - - pipeline.setName(pipelineName); - pipeline.setPipelineId(UUID.randomUUID().toString()); - - pipeline.setStreams(Collections.singletonList(prepareStream(datasetId))); - pipeline.setSepas(new ArrayList<>()); - pipeline.setActions(new ArrayList<>()); - collectInvocations("jsplumb_domId" + count, pipelineTemplateDescription.getBoundTo()); - - return pipeline; - } - - private SpDataStream prepareStream(String streamId) { - SpDataStream stream = getStream(streamId); - stream = new SpDataStream(stream); - stream.setDom(getDom()); - return stream; - } - - private void collectInvocations(String currentDomId, - List boundPipelineElements) { - for (BoundPipelineElement pipelineElement : boundPipelineElements) { - InvocableStreamPipesEntity entity = clonePe(pipelineElement.getPipelineElementTemplate()); - entity.setConnectedTo(Collections.singletonList(currentDomId)); - entity.setDom(getDom()); - if (entity instanceof DataProcessorInvocation) { - pipeline.getSepas().add((DataProcessorInvocation) entity); - if (pipelineElement.getConnectedTo().size() > 0) { - collectInvocations(entity.getDom(), pipelineElement.getConnectedTo()); - } - } else { - pipeline.getActions().add((DataSinkInvocation) entity); - } - } - PipelineModificationMessage message = new PipelineVerificationHandlerV2(pipeline).verifyPipeline(); - handleModifications(message); - } - - private void handleModifications(PipelineModificationMessage message) { - pipeline.getSepas().forEach(processor -> { - PipelineModification modification = getModification(message, processor.getDom()); - processor.setOutputStream(modification.getOutputStream()); - processor.setOutputStrategies(modification.getOutputStrategies()); - processor.setStaticProperties(modification.getStaticProperties()); - processor.setInputStreams(modification.getInputStreams()); - processor.setElementId(modification.getElementId()); - }); - pipeline.getActions().forEach(sink -> { - PipelineModification modification = getModification(message, sink.getDom()); - sink.setStaticProperties(modification.getStaticProperties()); - sink.setInputStreams(modification.getInputStreams()); - sink.setElementId(modification.getElementId()); - }); - } - - private PipelineModification getModification(PipelineModificationMessage message, - String domId) { - return message.getPipelineModifications() - .stream() - .filter(pm -> pm.getDomId().equals(domId)) - .findFirst() - .orElseThrow(IllegalArgumentException::new); - } - - private InvocableStreamPipesEntity clonePe(InvocableStreamPipesEntity pipelineElementTemplate) { - if (pipelineElementTemplate instanceof DataProcessorInvocation) { - return new DataProcessorInvocation((DataProcessorInvocation) pipelineElementTemplate); - } else { - return new DataSinkInvocation((DataSinkInvocation) pipelineElementTemplate); - } - } - - private SpDataStream getStream(String datasetId) { - return StorageDispatcher.INSTANCE.getNoSqlStore() - .getPipelineElementDescriptionStorage() - .getEventStreamById(datasetId); - } - - - private String getDom() { - count++; - return "jsplumb_domId" + count; - } -} diff --git a/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/template/PipelineTemplateGenerator.java b/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/template/PipelineTemplateGenerator.java deleted file mode 100644 index 86c46bc742..0000000000 --- a/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/template/PipelineTemplateGenerator.java +++ /dev/null @@ -1,75 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ -package org.apache.streampipes.manager.template; - -import org.apache.streampipes.commons.exceptions.ElementNotFoundException; -import org.apache.streampipes.manager.template.instances.DataLakePipelineTemplate; -import org.apache.streampipes.manager.template.instances.PipelineTemplate; -import org.apache.streampipes.model.graph.DataSinkDescription; -import org.apache.streampipes.model.template.PipelineTemplateDescription; -import org.apache.streampipes.storage.api.IPipelineElementDescriptionStorage; -import org.apache.streampipes.storage.management.StorageDispatcher; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.net.URISyntaxException; -import java.util.ArrayList; -import java.util.List; - -public class PipelineTemplateGenerator { - - - Logger logger = LoggerFactory.getLogger(PipelineTemplateGenerator.class); - - private final List availableDescriptions = new ArrayList<>(); - - public List getAllPipelineTemplates() { - - List allPipelineTemplates = new ArrayList<>(); - - allPipelineTemplates.add(new DataLakePipelineTemplate()); - - - for (PipelineTemplate pt : allPipelineTemplates) { - try { - availableDescriptions.add(pt.declareModel()); - } catch (URISyntaxException e) { - e.printStackTrace(); - } catch (ElementNotFoundException e) { - logger.warn("Adapter template can not be used because some elements are not installed", e); - } - } - - return availableDescriptions; - } - - protected DataSinkDescription getSink(String id) throws ElementNotFoundException { - try { - return getStorage() - .getDataSinkByAppId(id); - } catch (IllegalArgumentException e) { - throw new ElementNotFoundException("Data stream " + id + " is not installed!"); - } - } - - protected IPipelineElementDescriptionStorage getStorage() { - return StorageDispatcher.INSTANCE.getNoSqlStore() - .getPipelineElementDescriptionStorage(); - } -} diff --git a/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/template/PipelineTemplateInvocationGenerator.java b/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/template/PipelineTemplateInvocationGenerator.java deleted file mode 100644 index ac2e1ac221..0000000000 --- a/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/template/PipelineTemplateInvocationGenerator.java +++ /dev/null @@ -1,66 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ -package org.apache.streampipes.manager.template; - -import org.apache.streampipes.model.SpDataStream; -import org.apache.streampipes.model.pipeline.Pipeline; -import org.apache.streampipes.model.staticproperty.StaticProperty; -import org.apache.streampipes.model.template.PipelineTemplateDescription; -import org.apache.streampipes.model.template.PipelineTemplateInvocation; - -import java.util.ArrayList; -import java.util.List; - -public class PipelineTemplateInvocationGenerator { - - private final SpDataStream spDataStream; - private final PipelineTemplateDescription pipelineTemplateDescription; - - public PipelineTemplateInvocationGenerator(SpDataStream dataStream, - PipelineTemplateDescription pipelineTemplateDescription) { - this.spDataStream = dataStream; - this.pipelineTemplateDescription = pipelineTemplateDescription; - } - - public PipelineTemplateInvocation generateInvocation() { - - Pipeline pipeline = - new PipelineGenerator(spDataStream.getElementId(), pipelineTemplateDescription, "test").makePipeline(); - - PipelineTemplateInvocation pipelineTemplateInvocation = new PipelineTemplateInvocation(); - pipelineTemplateInvocation.setStaticProperties(collectStaticProperties(pipeline)); - pipelineTemplateInvocation.setDataStreamId(spDataStream.getElementId()); - pipelineTemplateInvocation.setPipelineTemplateId(pipelineTemplateDescription.getPipelineTemplateId()); - return pipelineTemplateInvocation; - } - - private List collectStaticProperties(Pipeline pipeline) { - List staticProperties = new ArrayList<>(); - - pipeline.getSepas().forEach(pe -> { - pe.getStaticProperties().forEach(sp -> sp.setInternalName(pe.getDom() + sp.getInternalName())); - staticProperties.addAll(pe.getStaticProperties()); - }); - pipeline.getActions().forEach(pe -> { - pe.getStaticProperties().forEach(sp -> sp.setInternalName(pe.getDom() + sp.getInternalName())); - staticProperties.addAll(pe.getStaticProperties()); - }); - - return staticProperties; - } -} diff --git a/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/template/PipelineTemplateInvocationHandler.java b/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/template/PipelineTemplateInvocationHandler.java deleted file mode 100644 index 7b1748a9ef..0000000000 --- a/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/template/PipelineTemplateInvocationHandler.java +++ /dev/null @@ -1,101 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ -package org.apache.streampipes.manager.template; - -import org.apache.streampipes.manager.execution.PipelineExecutor; -import org.apache.streampipes.manager.permission.PermissionManager; -import org.apache.streampipes.manager.storage.PipelineStorageService; -import org.apache.streampipes.model.base.InvocableStreamPipesEntity; -import org.apache.streampipes.model.client.user.Permission; -import org.apache.streampipes.model.pipeline.Pipeline; -import org.apache.streampipes.model.pipeline.PipelineOperationStatus; -import org.apache.streampipes.model.staticproperty.StaticProperty; -import org.apache.streampipes.model.template.PipelineTemplateDescription; -import org.apache.streampipes.model.template.PipelineTemplateInvocation; -import org.apache.streampipes.storage.management.StorageDispatcher; - -import java.util.ArrayList; -import java.util.List; - -public class PipelineTemplateInvocationHandler { - - private final PipelineTemplateInvocation pipelineTemplateInvocation; - private final PipelineTemplateDescription pipelineTemplateDescription; - private final String username; - - public PipelineTemplateInvocationHandler(String username, PipelineTemplateInvocation pipelineTemplateInvocation) { - this.username = username; - this.pipelineTemplateInvocation = pipelineTemplateInvocation; - this.pipelineTemplateDescription = getTemplateById(pipelineTemplateInvocation.getPipelineTemplateId()); - } - - public PipelineOperationStatus handlePipelineInvocation() { - Pipeline pipeline = new PipelineGenerator(pipelineTemplateInvocation.getDataStreamId(), pipelineTemplateDescription, - pipelineTemplateInvocation.getKviName()).makePipeline(); - pipeline.setCreatedByUser(username); - pipeline.setCreatedAt(System.currentTimeMillis()); - replaceStaticProperties(pipeline); - new PipelineStorageService(pipeline).addPipeline(); - Permission permission = new PermissionManager().makePermission(pipeline, username); - StorageDispatcher.INSTANCE.getNoSqlStore().getPermissionStorage().persist(permission); - Pipeline storedPipeline = - StorageDispatcher.INSTANCE.getNoSqlStore().getPipelineStorageAPI().getElementById(pipeline.getPipelineId()); - return new PipelineExecutor(storedPipeline).startPipeline(); - } - - private void replaceStaticProperties(Pipeline pipeline) { - pipeline.getSepas().forEach(this::replace); - pipeline.getActions().forEach(this::replace); - } - - private void replace(InvocableStreamPipesEntity pe) { - List newProperties = new ArrayList<>(); - pe.getStaticProperties().forEach(sp -> { - if (existsInCustomizedElements(pe.getDom(), sp)) { - newProperties.add(getCustomizedElement(pe.getDom(), pe.getDom() + sp.getInternalName())); - } else { - newProperties.add(sp); - } - }); - pe.setStaticProperties(newProperties); - } - - - private StaticProperty getCustomizedElement(String dom, String internalName) { - StaticProperty staticProperty = pipelineTemplateInvocation - .getStaticProperties() - .stream() - .filter(sp -> sp.getInternalName().equals(internalName)).findFirst().get(); - - staticProperty.setInternalName(staticProperty.getInternalName().replace(dom, "")); - return staticProperty; - } - - private boolean existsInCustomizedElements(String dom, StaticProperty staticProperty) { - return pipelineTemplateInvocation - .getStaticProperties() - .stream() - .anyMatch(sp -> sp.getInternalName().equals(dom + staticProperty.getInternalName())); - } - - - private PipelineTemplateDescription getTemplateById(String pipelineTemplateId) { - return new PipelineTemplateGenerator().getAllPipelineTemplates().stream() - .filter(template -> template.getAppId().equals(pipelineTemplateId)).findFirst().get(); - } -} diff --git a/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/template/PipelineTemplateManagement.java b/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/template/PipelineTemplateManagement.java deleted file mode 100644 index a561c5093b..0000000000 --- a/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/template/PipelineTemplateManagement.java +++ /dev/null @@ -1,80 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package org.apache.streampipes.manager.template; - -import org.apache.streampipes.model.SpDataStream; -import org.apache.streampipes.model.pipeline.PipelineOperationStatus; -import org.apache.streampipes.model.template.PipelineTemplateDescription; -import org.apache.streampipes.model.template.PipelineTemplateInvocation; -import org.apache.streampipes.storage.management.StorageDispatcher; - -import java.util.List; -import java.util.Optional; - -public class PipelineTemplateManagement { - - public PipelineTemplateInvocation prepareInvocation(String streamId, - String pipelineTemplateId) { - SpDataStream dataStream = getDataStream(streamId); - - var pipelineTemplateDescriptionOpt = getPipelineTemplateDescription(pipelineTemplateId); - if (pipelineTemplateDescriptionOpt.isPresent()) { - PipelineTemplateInvocation invocation = - new PipelineTemplateInvocationGenerator( - dataStream, - pipelineTemplateDescriptionOpt.get() - ).generateInvocation(); - PipelineTemplateInvocation clonedInvocation = new PipelineTemplateInvocation(invocation); - return new PipelineTemplateInvocation(clonedInvocation); - } else { - throw new IllegalArgumentException(String.format( - "Could not find pipeline template with ID %s", - pipelineTemplateId) - ); - } - } - - public PipelineOperationStatus createAndStartPipeline(PipelineTemplateInvocation pipelineTemplateInvocation, - String authenticatedUserSid) { - return new PipelineTemplateInvocationHandler( - authenticatedUserSid, - pipelineTemplateInvocation - ).handlePipelineInvocation(); - } - - private Optional getPipelineTemplateDescription(String pipelineTemplateId) { - return new PipelineTemplateGenerator() - .getAllPipelineTemplates() - .stream() - .filter(pt -> pt.getAppId().equals(pipelineTemplateId)) - .findFirst(); - } - - private SpDataStream getDataStream(String streamId) { - return getAllDataStreams() - .stream() - .filter(sp -> sp.getElementId().equals(streamId)) - .findFirst() - .get(); - } - - private List getAllDataStreams() { - return StorageDispatcher.INSTANCE.getNoSqlStore().getPipelineElementDescriptionStorage().getAllDataStreams(); - } -} diff --git a/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/template/compact/CompactPipelineTemplateManagement.java b/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/template/compact/CompactPipelineTemplateManagement.java new file mode 100644 index 0000000000..0bf308cfc4 --- /dev/null +++ b/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/template/compact/CompactPipelineTemplateManagement.java @@ -0,0 +1,98 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.streampipes.manager.template.compact; + +import org.apache.streampipes.manager.matching.PipelineVerificationHandlerV2; +import org.apache.streampipes.manager.pipeline.compact.generation.PipelineElementConfigurationStep; +import org.apache.streampipes.model.pipeline.Pipeline; +import org.apache.streampipes.model.pipeline.PipelineModificationResult; +import org.apache.streampipes.model.pipeline.compact.CompactPipelineElement; +import org.apache.streampipes.model.template.CompactPipelineTemplate; +import org.apache.streampipes.model.template.PipelineTemplateGenerationRequest; +import org.apache.streampipes.storage.api.CRUDStorage; +import org.apache.streampipes.storage.api.IPipelineElementDescriptionStorage; + +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.stream.Collectors; + +import static org.apache.streampipes.manager.pipeline.compact.generation.InvocablePipelineElementGenerator.ID_PREFIX; + +public class CompactPipelineTemplateManagement { + + private final IPipelineElementDescriptionStorage storage; + private final CRUDStorage templateStorage; + + public CompactPipelineTemplateManagement(CRUDStorage templateStorage, + IPipelineElementDescriptionStorage descriptionStorage) { + this.templateStorage = templateStorage; + this.storage = descriptionStorage; + } + + public PipelineModificationResult makePipeline(PipelineTemplateGenerationRequest request) throws Exception { + var template = request.template(); + if (request.streams() != null && !request.streams().isEmpty()) { + request.streams().forEach((key, value) -> { + var stream = storage.getDataStreamById(value); + template.getPipeline().add(new CompactPipelineElement( + "stream", + key, + stream.getElementId(), + List.of(), + List.of() + )); + }); + } + var pipeline = makePipeline(template); + + return new PipelineVerificationHandlerV2(pipeline).makeModifiedPipeline(); + } + + private Pipeline makePipeline(CompactPipelineTemplate template) throws Exception { + var pipeline = new Pipeline(); + template.getPipeline().forEach(pe -> { + List connectedToList = pe.connectedTo(); + Set validIds = template.getPipeline().stream() + .map(CompactPipelineElement::ref) + .collect(Collectors.toSet()); + connectedToList.removeIf(connectedId -> !validIds.contains(connectedId)); + }); + new PipelineElementConfigurationStep(storage).apply(pipeline, template.toCompactPipeline()); + + return pipeline; + } + + public Map>> getStreamsForTemplate(String pipelineTemplateId) throws Exception { + var template = Optional.ofNullable(templateStorage.getElementById(pipelineTemplateId)) + .orElseThrow(() -> new IllegalArgumentException("Template " + pipelineTemplateId + " not found")); + + var pipeline = makePipeline(template); + var allDataStreams = storage.getAllDataStreams(); + var requiredStreamInputs = template + .getPlaceholders() + .requiredStreamInputs() + .stream() + .map(c -> ID_PREFIX + c) + .toList(); + + return new MatchingStreamFinder().findMatchedStreams(pipeline, requiredStreamInputs, allDataStreams); + } +} diff --git a/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/template/compact/MatchingStreamFinder.java b/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/template/compact/MatchingStreamFinder.java new file mode 100644 index 0000000000..4bc5ef18f3 --- /dev/null +++ b/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/template/compact/MatchingStreamFinder.java @@ -0,0 +1,77 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.streampipes.manager.template.compact; + +import org.apache.streampipes.manager.matching.v2.SchemaMatch; +import org.apache.streampipes.model.SpDataStream; +import org.apache.streampipes.model.base.InvocableStreamPipesEntity; +import org.apache.streampipes.model.base.NamedStreamPipesEntity; +import org.apache.streampipes.model.pipeline.Pipeline; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Stream; + +public class MatchingStreamFinder { + + public Map>> findMatchedStreams(Pipeline pipeline, + List requiredStreamInputs, + List allDataStreams) { + var matchedStreams = new HashMap>>(); + + Stream.concat(pipeline.getSepas().stream(), pipeline.getActions().stream()) + .filter(pe -> hasRequiredInputs(pe, requiredStreamInputs)) + .forEach(pe -> matchedStreams.put( + pe.getElementId(), + findSupportedStreamsForPe(pe, requiredStreamInputs, allDataStreams)) + ); + + return matchedStreams; + } + + private boolean hasRequiredInputs(InvocableStreamPipesEntity pe, List requiredStreamInputs) { + return requiredStreamInputs + .stream() + .anyMatch(pe.getConnectedTo()::contains); + } + + private List> findSupportedStreamsForPe(InvocableStreamPipesEntity pe, + List requiredStreamInputs, + List allDataStreams) { + var streams = new ArrayList>(); + + for (int i = 0; i < pe.getConnectedTo().size(); i++) { + String connectedInput = pe.getConnectedTo().get(i); + if (requiredStreamInputs.contains(connectedInput)) { + streams.add(getSupportedStreams(allDataStreams, pe.getInputStreams().get(i))); + } + } + + return streams; + } + + private List getSupportedStreams(List allDataStreams, SpDataStream inputStreamReq) { + return allDataStreams.stream() + .filter(ds -> new SchemaMatch().match(ds.getEventSchema(), inputStreamReq.getEventSchema(), List.of())) + .map(NamedStreamPipesEntity::getElementId) + .toList(); + } +} diff --git a/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/template/instances/DataLakePipelineTemplate.java b/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/template/instances/DataLakePipelineTemplate.java deleted file mode 100644 index c52854a299..0000000000 --- a/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/template/instances/DataLakePipelineTemplate.java +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package org.apache.streampipes.manager.template.instances; - -import org.apache.streampipes.commons.exceptions.ElementNotFoundException; -import org.apache.streampipes.manager.template.PipelineTemplateGenerator; -import org.apache.streampipes.model.template.PipelineTemplateDescription; -import org.apache.streampipes.sdk.builder.BoundPipelineElementBuilder; -import org.apache.streampipes.sdk.builder.PipelineTemplateBuilder; - -public class DataLakePipelineTemplate extends PipelineTemplateGenerator implements PipelineTemplate { - - private static final String ID = "org.apache.streampipes.manager.template.instances.DataLakePipelineTemplate"; - - @Override - public PipelineTemplateDescription declareModel() throws ElementNotFoundException { - return new PipelineTemplateDescription( - PipelineTemplateBuilder.create("http://streampipes.org/DataLakePipelineTemplate", "DataLake", - "") - .setAppId(ID) - .boundPipelineElementTemplate( - BoundPipelineElementBuilder - .create(getSink("org.apache.streampipes.sinks.internal.jvm.datalake")) - .build()) - .build()); - - } -} diff --git a/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/template/instances/PipelineTemplate.java b/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/template/instances/DefaultPipelineTemplateProvider.java similarity index 73% rename from streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/template/instances/PipelineTemplate.java rename to streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/template/instances/DefaultPipelineTemplateProvider.java index 9e115cb212..2c3e53b8f3 100644 --- a/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/template/instances/PipelineTemplate.java +++ b/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/template/instances/DefaultPipelineTemplateProvider.java @@ -18,13 +18,9 @@ package org.apache.streampipes.manager.template.instances; -import org.apache.streampipes.commons.exceptions.ElementNotFoundException; -import org.apache.streampipes.model.template.PipelineTemplateDescription; +import org.apache.streampipes.model.template.CompactPipelineTemplate; -import java.net.URISyntaxException; - -public interface PipelineTemplate { - - PipelineTemplateDescription declareModel() throws URISyntaxException, ElementNotFoundException; +public interface DefaultPipelineTemplateProvider { + CompactPipelineTemplate getTemplate(); } diff --git a/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/template/instances/PersistDataLakePipelineTemplate.java b/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/template/instances/PersistDataLakePipelineTemplate.java new file mode 100644 index 0000000000..caa6e7bf94 --- /dev/null +++ b/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/template/instances/PersistDataLakePipelineTemplate.java @@ -0,0 +1,70 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.streampipes.manager.template.instances; + +import org.apache.streampipes.model.pipeline.compact.CompactPipelineElement; +import org.apache.streampipes.model.template.CompactPipelineTemplate; +import org.apache.streampipes.model.template.PipelinePlaceholderConfig; +import org.apache.streampipes.model.template.PipelinePlaceholders; + +import java.util.List; +import java.util.Map; + +public class PersistDataLakePipelineTemplate implements DefaultPipelineTemplateProvider { + + public static final String DATA_LAKE_SINK_REF = "lake"; + public static final String DATA_LAKE_SINK_ID = "org.apache.streampipes.sinks.internal.jvm.datalake"; + public static final String DATA_LAKE_CONNECTOR_ID = "stream1"; + public static final String DATA_LAKE_TEMPLATE_ID = "sp-internal-persist"; + + public static final String DATA_LAKE_MEASUREMENT_FIELD = "db_measurement"; + public static final String DATA_LAKE_TIMESTAMP_FIELD = "timestamp_mapping"; + public static final String DATA_LAKE_DIMENSIONS_FIELD = "dimensions_selection"; + + @Override + public CompactPipelineTemplate getTemplate() { + var template = new CompactPipelineTemplate(); + template.setElementId(DATA_LAKE_TEMPLATE_ID); + template.setName("Persist Data"); + template.setDescription("Use this template to persist an input data stream to the time-series storage"); + template.setPipeline(List.of( + new CompactPipelineElement( + "sink", + DATA_LAKE_SINK_REF, + DATA_LAKE_SINK_ID, + List.of(DATA_LAKE_CONNECTOR_ID), + List.of( + Map.of("schema_update", "Update schema"), + Map.of("ignore_duplicates", false) + ) + ) + ) + ); + template.setPlaceholders(new PipelinePlaceholders( + List.of("stream1"), + List.of( + new PipelinePlaceholderConfig(DATA_LAKE_SINK_REF, DATA_LAKE_MEASUREMENT_FIELD), + new PipelinePlaceholderConfig(DATA_LAKE_SINK_REF, DATA_LAKE_TIMESTAMP_FIELD), + new PipelinePlaceholderConfig(DATA_LAKE_SINK_REF, DATA_LAKE_DIMENSIONS_FIELD) + ) + )); + + return template; + } +} diff --git a/streampipes-rest-core-base/src/main/java/org/apache/streampipes/rest/core/base/impl/CRUDResource.java b/streampipes-rest-core-base/src/main/java/org/apache/streampipes/rest/core/base/impl/CRUDResource.java index 4a67fe1511..8e0818aac7 100644 --- a/streampipes-rest-core-base/src/main/java/org/apache/streampipes/rest/core/base/impl/CRUDResource.java +++ b/streampipes-rest-core-base/src/main/java/org/apache/streampipes/rest/core/base/impl/CRUDResource.java @@ -30,19 +30,38 @@ public interface CRUDResource { - @GetMapping(produces = MediaType.APPLICATION_JSON_VALUE) + @GetMapping(produces = { + MediaType.APPLICATION_JSON_VALUE, + "application/yaml", + "application/yml"}) List findAll(); - @GetMapping(path = "/{id}", produces = MediaType.APPLICATION_JSON_VALUE) + @GetMapping(path = "/{id}", produces = { + MediaType.APPLICATION_JSON_VALUE, + "application/yaml", + "application/yml"}) T findById(@PathVariable("id") String id); @PostMapping( - produces = MediaType.APPLICATION_JSON_VALUE, - consumes = MediaType.APPLICATION_JSON_VALUE + produces = { + MediaType.APPLICATION_JSON_VALUE, + "application/yaml", + "application/yml"}, + consumes = { + MediaType.APPLICATION_JSON_VALUE, + "application/yaml", + "application/yml"} ) void create(@RequestBody T entity); - @PutMapping(path = "/{id}", produces = MediaType.APPLICATION_JSON_VALUE, consumes = MediaType.APPLICATION_JSON_VALUE) + @PutMapping(path = "/{id}", produces = { + MediaType.APPLICATION_JSON_VALUE, + "application/yaml", + "application/yml" + }, consumes = { + MediaType.APPLICATION_JSON_VALUE, + "application/yaml", + "application/yml"}) ReT update(@RequestBody T entity); @DeleteMapping(path = "/{id}") diff --git a/streampipes-rest-shared/src/main/java/org/apache/streampipes/rest/shared/constants/SpMediaType.java b/streampipes-rest-shared/src/main/java/org/apache/streampipes/rest/shared/constants/SpMediaType.java new file mode 100644 index 0000000000..558f8dd721 --- /dev/null +++ b/streampipes-rest-shared/src/main/java/org/apache/streampipes/rest/shared/constants/SpMediaType.java @@ -0,0 +1,25 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.streampipes.rest.shared.constants; + +public class SpMediaType { + + public static final String YAML = "application/yaml"; + public static final String YML = "application/yml"; +} diff --git a/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/PipelineResource.java b/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/PipelineResource.java index aa362aba16..d2dbdf5676 100644 --- a/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/PipelineResource.java +++ b/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/PipelineResource.java @@ -22,6 +22,7 @@ import org.apache.streampipes.manager.execution.status.PipelineStatusManager; import org.apache.streampipes.manager.matching.PipelineVerificationHandlerV2; import org.apache.streampipes.manager.pipeline.PipelineManager; +import org.apache.streampipes.manager.pipeline.compact.CompactPipelineManagement; import org.apache.streampipes.manager.recommender.ElementRecommender; import org.apache.streampipes.manager.storage.PipelineStorageService; import org.apache.streampipes.model.message.ErrorMessage; @@ -34,10 +35,12 @@ import org.apache.streampipes.model.pipeline.Pipeline; import org.apache.streampipes.model.pipeline.PipelineElementRecommendationMessage; import org.apache.streampipes.model.pipeline.PipelineOperationStatus; +import org.apache.streampipes.model.pipeline.compact.CompactPipeline; import org.apache.streampipes.rest.core.base.impl.AbstractAuthGuardedRestResource; import org.apache.streampipes.rest.security.AuthConstants; import org.apache.streampipes.rest.shared.exception.SpMessageException; import org.apache.streampipes.rest.shared.exception.SpNotificationException; +import org.apache.streampipes.storage.management.StorageDispatcher; import com.google.gson.JsonSyntaxException; import io.swagger.v3.oas.annotations.Hidden; @@ -77,6 +80,14 @@ public class PipelineResource extends AbstractAuthGuardedRestResource { private static final Logger LOG = LoggerFactory.getLogger(PipelineResource.class); + private final CompactPipelineManagement compactPipelineManagement; + + public PipelineResource() { + this.compactPipelineManagement = new CompactPipelineManagement( + StorageDispatcher.INSTANCE.getNoSqlStore().getPipelineElementDescriptionStorage() + ); + } + @GetMapping(produces = MediaType.APPLICATION_JSON_VALUE) @Operation(summary = "Get all pipelines of the current user", tags = {"Pipeline"}, responses = { @ApiResponse(content = { @@ -164,6 +175,16 @@ public ResponseEntity addPipeline(@RequestBody Pipeline pipeline return ok(message); } + @PostMapping( + path = "compact", + consumes = MediaType.APPLICATION_JSON_VALUE, + produces = MediaType.APPLICATION_JSON_VALUE) + @Operation(summary = "Convert a pipeline to the compact model", tags = {"Pipeline"}) + @PreAuthorize(AuthConstants.HAS_WRITE_PIPELINE_PRIVILEGE) + public ResponseEntity convertToCompactPipeline(@RequestBody Pipeline pipeline) { + return ok(compactPipelineManagement.convertPipeline(pipeline)); + } + @PostMapping( path = "/recommend/{recId}", consumes = MediaType.APPLICATION_JSON_VALUE, diff --git a/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/PipelineTemplate.java b/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/PipelineTemplate.java index ee041f5232..ab36c1d0ed 100644 --- a/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/PipelineTemplate.java +++ b/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/PipelineTemplate.java @@ -15,79 +15,109 @@ * limitations under the License. * */ + package org.apache.streampipes.rest.impl; -import org.apache.streampipes.manager.template.PipelineTemplateManagement; -import org.apache.streampipes.model.SpDataStream; -import org.apache.streampipes.model.SpDataStreamContainer; -import org.apache.streampipes.model.message.Notifications; -import org.apache.streampipes.model.pipeline.PipelineOperationStatus; -import org.apache.streampipes.model.template.PipelineTemplateInvocation; +import org.apache.streampipes.manager.template.compact.CompactPipelineTemplateManagement; +import org.apache.streampipes.model.template.CompactPipelineTemplate; +import org.apache.streampipes.model.template.PipelineTemplateGenerationRequest; import org.apache.streampipes.rest.core.base.impl.AbstractAuthGuardedRestResource; -import org.apache.streampipes.rest.shared.exception.SpMessageException; +import org.apache.streampipes.rest.shared.constants.SpMediaType; +import org.apache.streampipes.rest.shared.exception.BadRequestException; +import org.apache.streampipes.storage.api.CRUDStorage; +import org.apache.streampipes.storage.management.StorageDispatcher; -import org.springframework.http.HttpStatus; import org.springframework.http.MediaType; import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.DeleteMapping; import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.PathVariable; import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.PutMapping; import org.springframework.web.bind.annotation.RequestBody; import org.springframework.web.bind.annotation.RequestMapping; -import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.bind.annotation.RestController; -import java.util.ArrayList; +import java.util.Comparator; import java.util.List; +import java.util.Map; @RestController @RequestMapping("/api/v2/pipeline-templates") public class PipelineTemplate extends AbstractAuthGuardedRestResource { - private final PipelineTemplateManagement pipelineTemplateManagement; + private final CRUDStorage storage; + private final CompactPipelineTemplateManagement templateManagement; public PipelineTemplate() { - this.pipelineTemplateManagement = new PipelineTemplateManagement(); + storage = StorageDispatcher.INSTANCE.getNoSqlStore().getPipelineTemplateStorage(); + templateManagement = new CompactPipelineTemplateManagement( + storage, + StorageDispatcher.INSTANCE.getNoSqlStore().getPipelineElementDescriptionStorage() + ); + } + + @GetMapping( + produces = {MediaType.APPLICATION_JSON_VALUE, SpMediaType.YAML, SpMediaType.YML}) + public List findAll() { + return storage.findAll() + .stream() + .sorted(Comparator.comparing(CompactPipelineTemplate::getName)) + .toList(); } - @GetMapping(path = "/streams", produces = MediaType.APPLICATION_JSON_VALUE) - public ResponseEntity getAvailableDataStreams() { - List sources = getPipelineElementRdfStorage().getAllDataStreams(); - List datasets = new ArrayList<>(); + @GetMapping( + path = "/{id}", + produces = {MediaType.APPLICATION_JSON_VALUE, SpMediaType.YAML, SpMediaType.YML}) + public ResponseEntity findById(@PathVariable("id") String id) { + return ok(storage.getElementById(id)); + } - sources.stream() - .map(SpDataStream::new) - .forEach(datasets::add); - return ok((new SpDataStreamContainer(datasets))); + @PostMapping( + produces = {MediaType.APPLICATION_JSON_VALUE, SpMediaType.YAML, SpMediaType.YML}, + consumes = {MediaType.APPLICATION_JSON_VALUE, SpMediaType.YAML, SpMediaType.YML}) + public void create(@RequestBody CompactPipelineTemplate entity) { + storage.persist(entity); } - @GetMapping( - path = "/invocation", - produces = MediaType.APPLICATION_JSON_VALUE) - public ResponseEntity getPipelineTemplateInvocation( - @RequestParam(value = "streamId", required = false) String streamId, - @RequestParam(value = "templateId") String pipelineTemplateId) { + @PutMapping(path = "/{id}", + produces = {MediaType.APPLICATION_JSON_VALUE, SpMediaType.YAML, SpMediaType.YML}, + consumes = {MediaType.APPLICATION_JSON_VALUE, SpMediaType.YAML, SpMediaType.YML}) + public void update(@PathVariable("id") String id, @RequestBody CompactPipelineTemplate entity) { + storage.updateElement(entity); + } + + @DeleteMapping(path = "/{id}") + public void delete(@PathVariable("id") String id) { + storage.deleteElementById(id); + } + + + @PostMapping(path = "/{id}/pipeline", + produces = {MediaType.APPLICATION_JSON_VALUE, SpMediaType.YAML, SpMediaType.YML}, + consumes = {MediaType.APPLICATION_JSON_VALUE, SpMediaType.YAML, SpMediaType.YML}) + public ResponseEntity makePipelineFromTemplate(@RequestBody PipelineTemplateGenerationRequest request) { try { - return ok(pipelineTemplateManagement.prepareInvocation(streamId, pipelineTemplateId)); + return ok(templateManagement.makePipeline(request).pipeline()); } catch (IllegalArgumentException e) { - throw new SpMessageException(HttpStatus.BAD_REQUEST, Notifications.error( - String.format( - "Could not create pipeline template %s - did you install all pipeline elements?", - pipelineTemplateId.substring(pipelineTemplateId.lastIndexOf(".") + 1)) - )); + return badRequest(e.getMessage()); + } catch (Exception e) { + throw new RuntimeException(e); } } - @PostMapping( - consumes = MediaType.APPLICATION_JSON_VALUE, - produces = MediaType.APPLICATION_JSON_VALUE) - public ResponseEntity generatePipeline( - @RequestBody PipelineTemplateInvocation pipelineTemplateInvocation) { - - var status = pipelineTemplateManagement.createAndStartPipeline( - pipelineTemplateInvocation, - getAuthenticatedUserSid() - ); - return ok(status); + @GetMapping( + path = "/{id}/streams", + produces = {MediaType.APPLICATION_JSON_VALUE, SpMediaType.YAML, SpMediaType.YML}) + public ResponseEntity>>> getAvailableStreamsForTemplate( + @PathVariable("id") String pipelineTemplateId) { + try { + return ok(templateManagement.getStreamsForTemplate(pipelineTemplateId)); + } catch (IllegalArgumentException e) { + throw new BadRequestException(e.getMessage()); + } catch (Exception e) { + throw new RuntimeException(e); + } } } diff --git a/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/connect/AdapterResource.java b/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/connect/AdapterResource.java index 0961a8c5c4..b972aaa7eb 100644 --- a/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/connect/AdapterResource.java +++ b/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/connect/AdapterResource.java @@ -22,11 +22,13 @@ import org.apache.streampipes.commons.prometheus.adapter.AdapterMetricsManager; import org.apache.streampipes.connect.management.management.AdapterMasterManagement; import org.apache.streampipes.connect.management.management.AdapterUpdateManagement; +import org.apache.streampipes.connect.management.management.CompactAdapterManagement; import org.apache.streampipes.manager.pipeline.PipelineManager; import org.apache.streampipes.model.client.user.DefaultRole; import org.apache.streampipes.model.client.user.Permission; import org.apache.streampipes.model.connect.adapter.AdapterDescription; import org.apache.streampipes.model.connect.adapter.PipelineUpdateInfo; +import org.apache.streampipes.model.connect.adapter.compact.CompactAdapter; import org.apache.streampipes.model.message.Message; import org.apache.streampipes.model.message.Notifications; import org.apache.streampipes.model.monitoring.SpLogMessage; @@ -34,6 +36,7 @@ import org.apache.streampipes.resource.management.PermissionResourceManager; import org.apache.streampipes.resource.management.SpResourceManager; import org.apache.streampipes.rest.security.AuthConstants; +import org.apache.streampipes.rest.shared.constants.SpMediaType; import org.apache.streampipes.storage.api.IPipelineStorage; import org.apache.streampipes.storage.management.StorageDispatcher; @@ -66,7 +69,7 @@ public class AdapterResource extends AbstractAdapterResource new AdapterMasterManagement( StorageDispatcher.INSTANCE.getNoSqlStore() - .getAdapterInstanceStorage(), + .getAdapterInstanceStorage(), new SpResourceManager().manageAdapters(), new SpResourceManager().manageDataStreams(), AdapterMetricsManager.INSTANCE.getAdapterMetrics() @@ -93,6 +96,17 @@ public ResponseEntity addAdapter(@RequestBody AdapterDescript return ok(Notifications.success(adapterId)); } + @PostMapping(path = "compact", consumes = {MediaType.APPLICATION_JSON_VALUE}, produces = { + MediaType.APPLICATION_JSON_VALUE, + SpMediaType.YAML, + SpMediaType.YML + }) + @PreAuthorize(AuthConstants.HAS_WRITE_ADAPTER_PRIVILEGE) + public ResponseEntity convertToCompactAdapter(@RequestBody AdapterDescription adapterDescription) + throws Exception { + return ok(new CompactAdapterManagement(List.of()).convertToCompactAdapter(adapterDescription)); + } + @PutMapping(produces = MediaType.APPLICATION_JSON_VALUE, consumes = MediaType.APPLICATION_JSON_VALUE) @PreAuthorize(AuthConstants.HAS_WRITE_ADAPTER_PRIVILEGE) public ResponseEntity updateAdapter(@RequestBody AdapterDescription adapterDescription) { @@ -108,7 +122,7 @@ public ResponseEntity updateAdapter(@RequestBody AdapterDescr } @PutMapping(path = "pipeline-migration-preflight", consumes = MediaType.APPLICATION_JSON_VALUE, - produces = MediaType.APPLICATION_JSON_VALUE) + produces = MediaType.APPLICATION_JSON_VALUE) @PreAuthorize(AuthConstants.HAS_WRITE_ADAPTER_PRIVILEGE) public ResponseEntity> performPipelineMigrationPreflight( @RequestBody AdapterDescription adapterDescription @@ -119,17 +133,26 @@ public ResponseEntity> performPipelineMigrationPrefligh return ok(migrations); } - @GetMapping(path = "/{id}", produces = { MediaType.APPLICATION_JSON_VALUE, "application/yaml" }) + @GetMapping(path = "/{id}", produces = {MediaType.APPLICATION_JSON_VALUE, SpMediaType.YAML, SpMediaType.YML}) @PreAuthorize(AuthConstants.HAS_READ_ADAPTER_PRIVILEGE) - public ResponseEntity getAdapter(@PathVariable("id") String adapterId) { + public ResponseEntity getAdapter(@PathVariable("id") String adapterId, + @RequestParam(value = "output", + defaultValue = "full", + required = false) String outputMode) { try { AdapterDescription adapterDescription = getAdapterDescription(adapterId); - - return ok(adapterDescription); + if (outputMode.equalsIgnoreCase("compact")) { + return ok(toCompactAdapterDescription(adapterDescription)); + } else { + return ok(adapterDescription); + } } catch (AdapterException e) { LOG.error("Error while getting adapter with id {}", adapterId, e); return fail(); + } catch (Exception e) { + LOG.error("Error while transforming adapter {}", adapterId, e); + return fail(); } } @@ -164,7 +187,7 @@ public ResponseEntity deleteAdapter(@PathVariable("id") String elementId, boolean deleteAssociatedPipelines) { List pipelinesUsingAdapter = getPipelinesUsingAdapter(elementId); IPipelineStorage pipelineStorageAPI = StorageDispatcher.INSTANCE.getNoSqlStore() - .getPipelineStorageAPI(); + .getPipelineStorageAPI(); if (pipelinesUsingAdapter.isEmpty()) { try { @@ -178,11 +201,11 @@ public ResponseEntity deleteAdapter(@PathVariable("id") String elementId, List namesOfPipelinesUsingAdapter = pipelinesUsingAdapter .stream() .map(pipelineId -> pipelineStorageAPI.getElementById( - pipelineId) - .getName()) + pipelineId) + .getName()) .collect(Collectors.toList()); return ResponseEntity.status(HttpStatus.SC_CONFLICT) - .body(String.join(", ", namesOfPipelinesUsingAdapter)); + .body(String.join(", ", namesOfPipelinesUsingAdapter)); } else { PermissionResourceManager permissionResourceManager = new PermissionResourceManager(); // find out the names of pipelines that have an owner and the owner is not the current user @@ -204,16 +227,16 @@ public ResponseEntity deleteAdapter(@PathVariable("id") String elementId, } managementService.deleteAdapter(elementId); return ok(Notifications.success("Adapter with id: " + elementId - + " and all pipelines using the adapter are deleted.")); + + " and all pipelines using the adapter are deleted.")); } catch (Exception e) { LOG.error("Error while deleting adapter with id " - + elementId + " and all pipelines using the adapter", e); + + elementId + " and all pipelines using the adapter", e); return ok(Notifications.error(e.getMessage())); } } else { // otherwise, hint the user the names of pipelines using the adapter but not owned by the user return ResponseEntity.status(HttpStatus.SC_CONFLICT) - .body(String.join(", ", namesOfPipelinesNotOwnedByUser)); + .body(String.join(", ", namesOfPipelinesNotOwnedByUser)); } } } @@ -226,7 +249,7 @@ public ResponseEntity getAllAdapters() { } catch (AdapterException e) { LOG.error("Error while getting all adapters", e); return ResponseEntity.status(500) - .build(); + .build(); } } @@ -234,10 +257,14 @@ private AdapterDescription getAdapterDescription(String adapterId) throws Adapte return managementService.getAdapter(adapterId); } + private CompactAdapter toCompactAdapterDescription(AdapterDescription adapterDescription) throws Exception { + return new CompactAdapterManagement(List.of()).convertToCompactAdapter(adapterDescription); + } + private List getPipelinesUsingAdapter(String adapterId) { return StorageDispatcher.INSTANCE.getNoSqlStore() - .getPipelineStorageAPI() - .getPipelinesUsingAdapter(adapterId); + .getPipelineStorageAPI() + .getPipelinesUsingAdapter(adapterId); } } diff --git a/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/connect/CompactAdapterResource.java b/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/connect/CompactAdapterResource.java index da76a0eff3..88424c2256 100644 --- a/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/connect/CompactAdapterResource.java +++ b/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/connect/CompactAdapterResource.java @@ -25,12 +25,13 @@ import org.apache.streampipes.connect.management.management.AdapterMasterManagement; import org.apache.streampipes.connect.management.management.AdapterUpdateManagement; import org.apache.streampipes.connect.management.management.CompactAdapterManagement; -import org.apache.streampipes.manager.template.PipelineTemplateManagement; +import org.apache.streampipes.manager.pipeline.compact.CompactPipelineManagement; import org.apache.streampipes.model.connect.adapter.AdapterDescription; import org.apache.streampipes.model.connect.adapter.compact.CompactAdapter; import org.apache.streampipes.model.message.Notifications; import org.apache.streampipes.resource.management.SpResourceManager; import org.apache.streampipes.rest.security.AuthConstants; +import org.apache.streampipes.rest.shared.constants.SpMediaType; import org.apache.streampipes.rest.shared.exception.BadRequestException; import org.apache.streampipes.storage.management.StorageDispatcher; @@ -69,8 +70,8 @@ public CompactAdapterResource() { @PostMapping( consumes = { MediaType.APPLICATION_JSON_VALUE, - "application/yaml", - "application/yml" + SpMediaType.YML, + SpMediaType.YAML } ) @PreAuthorize(AuthConstants.HAS_WRITE_ADAPTER_PRIVILEGE) @@ -88,7 +89,10 @@ public ResponseEntity addAdapterCompact( if (compactAdapter.createOptions().persist()) { var storedAdapter = managementService.getAdapter(adapterId); var status = new PersistPipelineHandler( - new PipelineTemplateManagement(), + getNoSqlStorage().getPipelineTemplateStorage(), + new CompactPipelineManagement( + getNoSqlStorage().getPipelineElementDescriptionStorage() + ), getAuthenticatedUserSid() ).createAndStartPersistPipeline(storedAdapter); } diff --git a/streampipes-sdk/src/main/java/org/apache/streampipes/sdk/builder/BoundPipelineElementBuilder.java b/streampipes-sdk/src/main/java/org/apache/streampipes/sdk/builder/BoundPipelineElementBuilder.java deleted file mode 100644 index 8382c9346c..0000000000 --- a/streampipes-sdk/src/main/java/org/apache/streampipes/sdk/builder/BoundPipelineElementBuilder.java +++ /dev/null @@ -1,63 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ -package org.apache.streampipes.sdk.builder; - -import org.apache.streampipes.model.base.InvocableStreamPipesEntity; -import org.apache.streampipes.model.graph.DataProcessorDescription; -import org.apache.streampipes.model.graph.DataProcessorInvocation; -import org.apache.streampipes.model.graph.DataSinkDescription; -import org.apache.streampipes.model.graph.DataSinkInvocation; -import org.apache.streampipes.model.template.BoundPipelineElement; - -import java.util.ArrayList; -import java.util.List; -import java.util.UUID; - -public class BoundPipelineElementBuilder { - - private BoundPipelineElement boundPipelineElement; - private InvocableStreamPipesEntity streamPipesEntity; - private List connectedTo; - - private BoundPipelineElementBuilder(InvocableStreamPipesEntity streamPipesEntity) { - this.streamPipesEntity = streamPipesEntity; - // TODO fix this hack - this.streamPipesEntity.setElementId(this.streamPipesEntity.getBelongsTo() + ":" + UUID.randomUUID().toString()); - this.boundPipelineElement = new BoundPipelineElement(); - this.connectedTo = new ArrayList<>(); - } - - public static BoundPipelineElementBuilder create(DataProcessorDescription dataProcessorDescription) { - return new BoundPipelineElementBuilder(new DataProcessorInvocation(dataProcessorDescription)); - } - - public static BoundPipelineElementBuilder create(DataSinkDescription dataSinkDescription) { - return new BoundPipelineElementBuilder(new DataSinkInvocation(dataSinkDescription)); - } - - public BoundPipelineElementBuilder connectTo(BoundPipelineElement boundPipelineElement) { - this.connectedTo.add(boundPipelineElement); - return this; - } - - public BoundPipelineElement build() { - this.boundPipelineElement.setPipelineElementTemplate(streamPipesEntity); - this.boundPipelineElement.setConnectedTo(connectedTo); - return boundPipelineElement; - } -} diff --git a/streampipes-sdk/src/main/java/org/apache/streampipes/sdk/builder/PipelineTemplateBuilder.java b/streampipes-sdk/src/main/java/org/apache/streampipes/sdk/builder/PipelineTemplateBuilder.java deleted file mode 100644 index 432e2fd5c6..0000000000 --- a/streampipes-sdk/src/main/java/org/apache/streampipes/sdk/builder/PipelineTemplateBuilder.java +++ /dev/null @@ -1,61 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ -package org.apache.streampipes.sdk.builder; - -import org.apache.streampipes.model.template.BoundPipelineElement; -import org.apache.streampipes.model.template.PipelineTemplateDescription; - -import java.util.ArrayList; -import java.util.List; - -public class PipelineTemplateBuilder { - - private PipelineTemplateDescription pipelineTemplateDescription; - private List boundPipelineElements; - private String appId; - - private PipelineTemplateBuilder(String internalId, String pipelineTemplateName, String pipelineTemplateDescription) { - this.pipelineTemplateDescription = new PipelineTemplateDescription(); - this.pipelineTemplateDescription.setPipelineTemplateName(pipelineTemplateName); - this.pipelineTemplateDescription.setPipelineTemplateId(internalId); - this.pipelineTemplateDescription.setPipelineTemplateDescription(pipelineTemplateDescription); - this.boundPipelineElements = new ArrayList<>(); - } - - public static PipelineTemplateBuilder create(String internalId, String pipelineTemplateName, - String pipelineTemplateDescription) { - return new PipelineTemplateBuilder(internalId, pipelineTemplateName, pipelineTemplateDescription); - } - - public PipelineTemplateBuilder setAppId(String id) { - this.pipelineTemplateDescription.setAppId(id); - return this; - } - - - public PipelineTemplateBuilder boundPipelineElementTemplate(BoundPipelineElement boundPipelineElement) { - this.boundPipelineElements.add(boundPipelineElement); - return this; - } - - public PipelineTemplateDescription build() { - this.pipelineTemplateDescription.setBoundTo(boundPipelineElements); - return pipelineTemplateDescription; - } - -} diff --git a/streampipes-service-core/src/main/java/org/apache/streampipes/service/core/migrations/AvailableMigrations.java b/streampipes-service-core/src/main/java/org/apache/streampipes/service/core/migrations/AvailableMigrations.java index fda4cd49ec..ed731d7ce8 100644 --- a/streampipes-service-core/src/main/java/org/apache/streampipes/service/core/migrations/AvailableMigrations.java +++ b/streampipes-service-core/src/main/java/org/apache/streampipes/service/core/migrations/AvailableMigrations.java @@ -26,6 +26,7 @@ import org.apache.streampipes.service.core.migrations.v093.AdapterMigration; import org.apache.streampipes.service.core.migrations.v093.StoreEmailTemplatesMigration; import org.apache.streampipes.service.core.migrations.v095.MergeFilenamesAndRenameDuplicatesMigration; +import org.apache.streampipes.service.core.migrations.v970.AddDataLakePipelineTemplateMigration; import org.apache.streampipes.service.core.migrations.v970.AddLinkSettingsMigration; import org.apache.streampipes.service.core.migrations.v970.AddRolesToUserDbMigration; import org.apache.streampipes.service.core.migrations.v970.DataExplorerDataViewMigration; @@ -50,7 +51,8 @@ public List getAvailableMigrations() { new DataExplorerDataViewMigration(), new ModifyAssetLinkTypeMigration(), new RemoveNodesFromOpcUaAdaptersMigration(), - new AddRolesToUserDbMigration() + new AddRolesToUserDbMigration(), + new AddDataLakePipelineTemplateMigration() ); } } diff --git a/streampipes-service-core/src/main/java/org/apache/streampipes/service/core/migrations/v970/AddDataLakePipelineTemplateMigration.java b/streampipes-service-core/src/main/java/org/apache/streampipes/service/core/migrations/v970/AddDataLakePipelineTemplateMigration.java new file mode 100644 index 0000000000..fd7fced44b --- /dev/null +++ b/streampipes-service-core/src/main/java/org/apache/streampipes/service/core/migrations/v970/AddDataLakePipelineTemplateMigration.java @@ -0,0 +1,51 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.streampipes.service.core.migrations.v970; + +import org.apache.streampipes.manager.template.instances.PersistDataLakePipelineTemplate; +import org.apache.streampipes.model.template.CompactPipelineTemplate; +import org.apache.streampipes.service.core.migrations.Migration; +import org.apache.streampipes.storage.api.CRUDStorage; +import org.apache.streampipes.storage.management.StorageDispatcher; + +import java.io.IOException; + +public class AddDataLakePipelineTemplateMigration implements Migration { + + private final CRUDStorage storage; + + public AddDataLakePipelineTemplateMigration() { + this.storage = StorageDispatcher.INSTANCE.getNoSqlStore().getPipelineTemplateStorage(); + } + + @Override + public boolean shouldExecute() { + return storage.getElementById(PersistDataLakePipelineTemplate.DATA_LAKE_TEMPLATE_ID) == null; + } + + @Override + public void executeMigration() throws IOException { + storage.persist(new PersistDataLakePipelineTemplate().getTemplate()); + } + + @Override + public String getDescription() { + return "Adding data lake pipeline template to template storage"; + } +} diff --git a/streampipes-storage-api/src/main/java/org/apache/streampipes/storage/api/INoSqlStorage.java b/streampipes-storage-api/src/main/java/org/apache/streampipes/storage/api/INoSqlStorage.java index 482051e353..0a64296929 100644 --- a/streampipes-storage-api/src/main/java/org/apache/streampipes/storage/api/INoSqlStorage.java +++ b/streampipes-storage-api/src/main/java/org/apache/streampipes/storage/api/INoSqlStorage.java @@ -29,6 +29,7 @@ import org.apache.streampipes.model.extensions.configuration.SpServiceConfiguration; import org.apache.streampipes.model.extensions.svcdiscovery.SpServiceRegistration; import org.apache.streampipes.model.file.FileMetadata; +import org.apache.streampipes.model.template.CompactPipelineTemplate; public interface INoSqlStorage { @@ -87,4 +88,6 @@ public interface INoSqlStorage { CRUDStorage getRoleStorage(); CRUDStorage getPrivilegeStorage(); + + CRUDStorage getPipelineTemplateStorage(); } diff --git a/streampipes-storage-couchdb/src/main/java/org/apache/streampipes/storage/couchdb/CouchDbStorageManager.java b/streampipes-storage-couchdb/src/main/java/org/apache/streampipes/storage/couchdb/CouchDbStorageManager.java index ab618c9512..ab2ce7e4b1 100644 --- a/streampipes-storage-couchdb/src/main/java/org/apache/streampipes/storage/couchdb/CouchDbStorageManager.java +++ b/streampipes-storage-couchdb/src/main/java/org/apache/streampipes/storage/couchdb/CouchDbStorageManager.java @@ -29,6 +29,7 @@ import org.apache.streampipes.model.extensions.configuration.SpServiceConfiguration; import org.apache.streampipes.model.extensions.svcdiscovery.SpServiceRegistration; import org.apache.streampipes.model.file.FileMetadata; +import org.apache.streampipes.model.template.CompactPipelineTemplate; import org.apache.streampipes.storage.api.CRUDStorage; import org.apache.streampipes.storage.api.IAdapterStorage; import org.apache.streampipes.storage.api.IDataProcessorStorage; @@ -245,4 +246,12 @@ public CRUDStorage getRoleStorage() { public CRUDStorage getPrivilegeStorage() { return new PrivilegeStorageImpl(); } + + @Override + public CRUDStorage getPipelineTemplateStorage() { + return new DefaultCrudStorage<>( + () -> Utils.getCouchDbGsonClient("pipeline-templates"), + CompactPipelineTemplate.class + ); + } } diff --git a/streampipes-wrapper-standalone/src/main/java/org/apache/streampipes/wrapper/standalone/StreamPipesNotificationSink.java b/streampipes-wrapper-standalone/src/main/java/org/apache/streampipes/wrapper/standalone/StreamPipesNotificationSink.java index eaa0917287..d4fc478e55 100644 --- a/streampipes-wrapper-standalone/src/main/java/org/apache/streampipes/wrapper/standalone/StreamPipesNotificationSink.java +++ b/streampipes-wrapper-standalone/src/main/java/org/apache/streampipes/wrapper/standalone/StreamPipesNotificationSink.java @@ -84,7 +84,7 @@ public void onInvocation( // convert input given in minutes to seconds // this is later used to determine if a notification should be sent this.silentPeriodInSeconds = parameters.extractor() - .singleValueParameter(KEY_SILENT_PERIOD, Long.class) * 60 + .singleValueParameter(KEY_SILENT_PERIOD, Integer.class) * 60 ; } diff --git a/ui/projects/streampipes/platform-services/src/lib/apis/adapter.service.ts b/ui/projects/streampipes/platform-services/src/lib/apis/adapter.service.ts index a18160a427..ff32b390f4 100644 --- a/ui/projects/streampipes/platform-services/src/lib/apis/adapter.service.ts +++ b/ui/projects/streampipes/platform-services/src/lib/apis/adapter.service.ts @@ -24,6 +24,7 @@ import { Observable } from 'rxjs'; import { PlatformServicesCommons } from './commons.service'; import { AdapterDescription, + CompactAdapter, Message, PipelineUpdateInfo, } from '../model/gen/streampipes-model'; @@ -55,6 +56,15 @@ export class AdapterService { ); } + convertToCompactAdapter( + adapterDescription: AdapterDescription, + ): Observable { + return this.http.post( + this.connectPath + `/master/adapters/compact`, + adapterDescription, + ); + } + requestAdapterDescriptions(path: string): Observable { return this.http.get(this.connectPath + path).pipe( map(response => { diff --git a/ui/projects/streampipes/platform-services/src/lib/apis/compact-pipeline.service.ts b/ui/projects/streampipes/platform-services/src/lib/apis/compact-pipeline.service.ts new file mode 100644 index 0000000000..49e3d5d4da --- /dev/null +++ b/ui/projects/streampipes/platform-services/src/lib/apis/compact-pipeline.service.ts @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +import { Injectable } from '@angular/core'; +import { HttpClient } from '@angular/common/http'; +import { Observable } from 'rxjs'; +import { PlatformServicesCommons } from './commons.service'; +import { CompactPipeline } from '../model/gen/streampipes-model'; + +@Injectable({ + providedIn: 'root', +}) +export class CompactPipelineService { + constructor( + private http: HttpClient, + private platformServicesCommons: PlatformServicesCommons, + ) {} + + create(pipeline: CompactPipeline): Observable { + return this.http.post(this.baseUrl, pipeline); + } + + get baseUrl(): string { + return `${this.platformServicesCommons.apiBasePath}/compact-pipelines`; + } +} diff --git a/ui/projects/streampipes/platform-services/src/lib/apis/pipeline-template.service.ts b/ui/projects/streampipes/platform-services/src/lib/apis/pipeline-template.service.ts index 2df674c729..abd4ccf764 100644 --- a/ui/projects/streampipes/platform-services/src/lib/apis/pipeline-template.service.ts +++ b/ui/projects/streampipes/platform-services/src/lib/apis/pipeline-template.service.ts @@ -19,10 +19,9 @@ import { Injectable } from '@angular/core'; import { HttpClient } from '@angular/common/http'; import { - FreeTextStaticProperty, - PipelineOperationStatus, - PipelineTemplateInvocation, - StaticPropertyUnion, + CompactPipelineTemplate, + Pipeline, + PipelineTemplateGenerationRequest, } from '../model/gen/streampipes-model'; import { map } from 'rxjs/operators'; import { Observable } from 'rxjs'; @@ -37,96 +36,24 @@ export class PipelineTemplateService { return '/streampipes-backend'; } - // getDataSets(): Observable { - // return this.http - // .get(this.getServerUrl() + '/api/v2/users/'+ this.authStatusService.email + '/pipeline-templates/streams') - // .pipe(map(response => { - // - // - // - // // TODO remove this - // // quick fix to deserialize URIs - // response['@graph'].forEach(function (object) { - // if (object['sp:domainProperty'] != undefined) { - // // object['sp:domainProperty']['@type'] = "sp:URI"; - // object['sp:domainProperty'] = object['sp:domainProperty']['@id']; - // delete object['sp:domainProperty']['@id']; - // } - // }); - // - // const res = this.tsonLdSerializerService.fromJsonLd(response, 'sp:DataStreamContainer'); - // return res.list; - // })); - // } - // - // getOperators(dataSet: DataSetDescription): Observable { - // return this.http - // .get(this.getServerUrl() + '/api/v2/users/'+ this.authStatusService.email + '/pipeline-templates?dataset=' + dataSet.id) - // .pipe(map(response => { - // const res = this.tsonLdSerializerService.fromJsonLd(response, 'sp:PipelineTemplateDescriptionContainer'); - // return res.list; - // })); - // } - - getPipelineTemplateInvocation( - dataSetId: string, - templateId: string, - ): Observable { - return this.http - .get( - `${this.getServerUrl()}/api/v2/pipeline-templates/invocation?streamId=${dataSetId}&templateId=${templateId}`, - ) - .pipe( - map(data => { - return PipelineTemplateInvocation.fromData( - data as PipelineTemplateInvocation, - ); - }), - ); - - // .pipe(map(response: PipelineTemplateInvocation => { - - // Currently tsonld dows not support objects that just contain one root object without an enclosing @graph array - // const res = new PipelineTemplateInvocation(response['@id']); - // res.dataSetId = response['sp:hasDataSetId']; - // res.name = response['hasElementName']; - // res.pipelineTemplateId = response['sp:hasInternalName']; - - // TODO find better solution - // This will remove preconfigured values from the UI - // res.list.forEach(property => { - // if (this.isFreeTextStaticProperty(property)) { - // if (this.asFreeTextStaticProperty(property).value !== undefined) { - // this.asFreeTextStaticProperty(property).render = false; - // } - // } - // }); - // return res; - // })); + findAll(): Observable { + return this.http.get(`${this.baseUrl}`); } - isFreeTextStaticProperty(val) { - return val instanceof FreeTextStaticProperty; + findById(id: string): Observable { + return this.http.get(`${this.baseUrl}/${id}`); } - asFreeTextStaticProperty(val: StaticPropertyUnion): FreeTextStaticProperty { - return val as FreeTextStaticProperty; + getPipelineForTemplate( + id: string, + request: PipelineTemplateGenerationRequest, + ): Observable { + return this.http + .post(`${this.baseUrl}/${id}/pipeline`, request) + .pipe(map(res => Pipeline.fromData(res as any))); } - createPipelineTemplateInvocation( - invocation: PipelineTemplateInvocation, - ): Observable { - return this.http - .post( - `${this.getServerUrl()}/api/v2/pipeline-templates`, - invocation, - ) - .pipe( - map(result => - PipelineOperationStatus.fromData( - result as PipelineOperationStatus, - ), - ), - ); + get baseUrl(): string { + return `${this.getServerUrl()}/api/v2/pipeline-templates`; } } diff --git a/ui/projects/streampipes/platform-services/src/lib/apis/pipeline.service.ts b/ui/projects/streampipes/platform-services/src/lib/apis/pipeline.service.ts index d4e084424c..de5c37517e 100644 --- a/ui/projects/streampipes/platform-services/src/lib/apis/pipeline.service.ts +++ b/ui/projects/streampipes/platform-services/src/lib/apis/pipeline.service.ts @@ -21,6 +21,7 @@ import { HttpClient } from '@angular/common/http'; import { PlatformServicesCommons } from './commons.service'; import { Observable } from 'rxjs'; import { + CompactPipeline, Message, Pipeline, PipelineElementRecommendationMessage, @@ -75,6 +76,13 @@ export class PipelineService { .pipe(map(response => Pipeline.fromData(response as Pipeline))); } + convertToCompactPipeline(pipeline: Pipeline): Observable { + return this.http.post( + `${this.apiBasePath}/pipelines/compact`, + pipeline, + ); + } + storePipeline(pipeline: Pipeline): Observable { return this.http.post(`${this.apiBasePath}/pipelines`, pipeline).pipe( map(response => { diff --git a/ui/projects/streampipes/platform-services/src/lib/model/gen/streampipes-model.ts b/ui/projects/streampipes/platform-services/src/lib/model/gen/streampipes-model.ts index e5faa9118d..cf07248af8 100644 --- a/ui/projects/streampipes/platform-services/src/lib/model/gen/streampipes-model.ts +++ b/ui/projects/streampipes/platform-services/src/lib/model/gen/streampipes-model.ts @@ -16,14 +16,14 @@ * specific language governing permissions and limitations * under the License. */ + /* tslint:disable */ /* eslint-disable */ // @ts-nocheck -// Generated using typescript-generator version 3.2.1263 on 2024-10-07 13:58:27. +// Generated using typescript-generator version 3.2.1263 on 2024-10-11 10:41:46. export class NamedStreamPipesEntity implements Storable { '@class': - | 'org.apache.streampipes.model.template.PipelineTemplateDescription' | 'org.apache.streampipes.model.SpDataStream' | 'org.apache.streampipes.model.base.VersionedNamedStreamPipesEntity' | 'org.apache.streampipes.model.connect.adapter.AdapterDescription' @@ -649,28 +649,6 @@ export class AssetExportConfiguration { } } -export class BoundPipelineElement { - connectedTo: BoundPipelineElement[]; - pipelineElementTemplate: InvocableStreamPipesEntity; - - static fromData( - data: BoundPipelineElement, - target?: BoundPipelineElement, - ): BoundPipelineElement { - if (!data) { - return data; - } - const instance = target || new BoundPipelineElement(); - instance.connectedTo = __getCopyArrayFn(BoundPipelineElement.fromData)( - data.connectedTo, - ); - instance.pipelineElementTemplate = InvocableStreamPipesEntity.fromData( - data.pipelineElementTemplate, - ); - return instance; - } -} - export class CanvasPosition { x: number; y: number; @@ -778,6 +756,148 @@ export class ColorPickerStaticProperty extends StaticProperty { } } +export class CompactAdapter { + appId: string; + configuration: { [index: string]: any }[]; + createOptions: CreateOptions; + description: string; + enrich: EnrichmentConfig; + id: string; + name: string; + schema: { [index: string]: CompactEventProperty }; + transform: TransformationConfig; + + static fromData( + data: CompactAdapter, + target?: CompactAdapter, + ): CompactAdapter { + if (!data) { + return data; + } + const instance = target || new CompactAdapter(); + instance.appId = data.appId; + instance.configuration = __getCopyArrayFn( + __getCopyObjectFn(__identity()), + )(data.configuration); + instance.createOptions = CreateOptions.fromData(data.createOptions); + instance.description = data.description; + instance.enrich = EnrichmentConfig.fromData(data.enrich); + instance.id = data.id; + instance.name = data.name; + instance.schema = __getCopyObjectFn(CompactEventProperty.fromData)( + data.schema, + ); + instance.transform = TransformationConfig.fromData(data.transform); + return instance; + } +} + +export class CompactEventProperty { + description: string; + label: string; + propertyScope: string; + semanticType: string; + + static fromData( + data: CompactEventProperty, + target?: CompactEventProperty, + ): CompactEventProperty { + if (!data) { + return data; + } + const instance = target || new CompactEventProperty(); + instance.description = data.description; + instance.label = data.label; + instance.propertyScope = data.propertyScope; + instance.semanticType = data.semanticType; + return instance; + } +} + +export class CompactPipeline { + createOptions: CreateOptions; + description: string; + id: string; + name: string; + pipelineElements: CompactPipelineElement[]; + + static fromData( + data: CompactPipeline, + target?: CompactPipeline, + ): CompactPipeline { + if (!data) { + return data; + } + const instance = target || new CompactPipeline(); + instance.createOptions = CreateOptions.fromData(data.createOptions); + instance.description = data.description; + instance.id = data.id; + instance.name = data.name; + instance.pipelineElements = __getCopyArrayFn( + CompactPipelineElement.fromData, + )(data.pipelineElements); + return instance; + } +} + +export class CompactPipelineElement { + configuration: { [index: string]: any }[]; + connectedTo: string[]; + id: string; + ref: string; + type: string; + + static fromData( + data: CompactPipelineElement, + target?: CompactPipelineElement, + ): CompactPipelineElement { + if (!data) { + return data; + } + const instance = target || new CompactPipelineElement(); + instance.configuration = __getCopyArrayFn( + __getCopyObjectFn(__identity()), + )(data.configuration); + instance.connectedTo = __getCopyArrayFn(__identity())( + data.connectedTo, + ); + instance.id = data.id; + instance.ref = data.ref; + instance.type = data.type; + return instance; + } +} + +export class CompactPipelineTemplate implements Storable { + description: string; + elementId: string; + name: string; + pipeline: CompactPipelineElement[]; + placeholders: PipelinePlaceholders; + rev: string; + + static fromData( + data: CompactPipelineTemplate, + target?: CompactPipelineTemplate, + ): CompactPipelineTemplate { + if (!data) { + return data; + } + const instance = target || new CompactPipelineTemplate(); + instance.description = data.description; + instance.elementId = data.elementId; + instance.name = data.name; + instance.pipeline = __getCopyArrayFn(CompactPipelineElement.fromData)( + data.pipeline, + ); + instance.placeholders = PipelinePlaceholders.fromData( + data.placeholders, + ); + instance.rev = data.rev; + return instance; + } +} + export class ConfigItem { configurationScope: ConfigurationScope; description: string; @@ -904,6 +1024,24 @@ export class CreateNestedRuleDescription extends SchemaTransformationRuleDescrip } } +export class CreateOptions { + persist: boolean; + start: boolean; + + static fromData( + data: CreateOptions, + target?: CreateOptions, + ): CreateOptions { + if (!data) { + return data; + } + const instance = target || new CreateOptions(); + instance.persist = data.persist; + instance.start = data.start; + return instance; + } +} + export class CustomOutputStrategy extends OutputStrategy { '@class': 'org.apache.streampipes.model.output.CustomOutputStrategy'; 'availablePropertyKeys': string[]; @@ -1381,6 +1519,22 @@ export interface EndpointSelectable { selectedEndpointUrl: string; } +export class EnrichmentConfig { + timestamp: string; + + static fromData( + data: EnrichmentConfig, + target?: EnrichmentConfig, + ): EnrichmentConfig { + if (!data) { + return data; + } + const instance = target || new EnrichmentConfig(); + instance.timestamp = data.timestamp; + return instance; + } +} + export class ValueSpecification { '@class': | 'org.apache.streampipes.model.schema.QuantitativeValue' @@ -2845,6 +2999,46 @@ export class PipelineOperationStatus { } } +export class PipelinePlaceholderConfig { + id: string; + ref: string; + + static fromData( + data: PipelinePlaceholderConfig, + target?: PipelinePlaceholderConfig, + ): PipelinePlaceholderConfig { + if (!data) { + return data; + } + const instance = target || new PipelinePlaceholderConfig(); + instance.id = data.id; + instance.ref = data.ref; + return instance; + } +} + +export class PipelinePlaceholders { + requiredConfigs: PipelinePlaceholderConfig[]; + requiredStreamInputs: string[]; + + static fromData( + data: PipelinePlaceholders, + target?: PipelinePlaceholders, + ): PipelinePlaceholders { + if (!data) { + return data; + } + const instance = target || new PipelinePlaceholders(); + instance.requiredConfigs = __getCopyArrayFn( + PipelinePlaceholderConfig.fromData, + )(data.requiredConfigs); + instance.requiredStreamInputs = __getCopyArrayFn(__identity())( + data.requiredStreamInputs, + ); + return instance; + } +} + export class PipelinePreviewModel { elementIdMappings: { [index: string]: string }; previewId: string; @@ -2887,59 +3081,26 @@ export class PipelineStatusMessage { } } -export class PipelineTemplateDescription extends NamedStreamPipesEntity { - '@class': 'org.apache.streampipes.model.template.PipelineTemplateDescription'; - 'boundTo': BoundPipelineElement[]; - 'pipelineTemplateDescription': string; - 'pipelineTemplateId': string; - 'pipelineTemplateName': string; +export class PipelineTemplateGenerationRequest { + pipelineDescription: string; + pipelineName: string; + streams: { [index: string]: string }; + template: CompactPipelineTemplate; - static 'fromData'( - data: PipelineTemplateDescription, - target?: PipelineTemplateDescription, - ): PipelineTemplateDescription { + static fromData( + data: PipelineTemplateGenerationRequest, + target?: PipelineTemplateGenerationRequest, + ): PipelineTemplateGenerationRequest { if (!data) { return data; } - const instance = target || new PipelineTemplateDescription(); - super.fromData(data, instance); - instance.boundTo = __getCopyArrayFn(BoundPipelineElement.fromData)( - data.boundTo, + const instance = target || new PipelineTemplateGenerationRequest(); + instance.pipelineDescription = data.pipelineDescription; + instance.pipelineName = data.pipelineName; + instance.streams = __getCopyObjectFn(__identity())( + data.streams, ); - instance.pipelineTemplateDescription = data.pipelineTemplateDescription; - instance.pipelineTemplateId = data.pipelineTemplateId; - instance.pipelineTemplateName = data.pipelineTemplateName; - return instance; - } -} - -export class PipelineTemplateInvocation { - '@class': 'org.apache.streampipes.model.template.PipelineTemplateInvocation'; - 'dataStreamId': string; - 'kviName': string; - 'pipelineTemplateDescription': PipelineTemplateDescription; - 'pipelineTemplateId': string; - 'staticProperties': StaticPropertyUnion[]; - - static 'fromData'( - data: PipelineTemplateInvocation, - target?: PipelineTemplateInvocation, - ): PipelineTemplateInvocation { - if (!data) { - return data; - } - const instance = target || new PipelineTemplateInvocation(); - instance['@class'] = data['@class']; - instance.dataStreamId = data.dataStreamId; - instance.kviName = data.kviName; - instance.pipelineTemplateDescription = - PipelineTemplateDescription.fromData( - data.pipelineTemplateDescription, - ); - instance.pipelineTemplateId = data.pipelineTemplateId; - instance.staticProperties = __getCopyArrayFn( - StaticProperty.fromDataUnion, - )(data.staticProperties); + instance.template = CompactPipelineTemplate.fromData(data.template); return instance; } } @@ -3823,6 +3984,26 @@ export class TransformOutputStrategy extends OutputStrategy { } } +export class TransformationConfig { + measurementUnit: { [index: string]: string }; + rename: { [index: string]: string }; + + static fromData( + data: TransformationConfig, + target?: TransformationConfig, + ): TransformationConfig { + if (!data) { + return data; + } + const instance = target || new TransformationConfig(); + instance.measurementUnit = __getCopyObjectFn(__identity())( + data.measurementUnit, + ); + instance.rename = __getCopyObjectFn(__identity())(data.rename); + return instance; + } +} + export class TreeInputNode { children: TreeInputNode[]; dataNode: boolean; diff --git a/ui/projects/streampipes/platform-services/src/public-api.ts b/ui/projects/streampipes/platform-services/src/public-api.ts index be270b9fa6..caa7068565 100644 --- a/ui/projects/streampipes/platform-services/src/public-api.ts +++ b/ui/projects/streampipes/platform-services/src/public-api.ts @@ -26,6 +26,7 @@ export * from './lib/apis/commons.service'; export * from './lib/apis/adapter.service'; export * from './lib/apis/adapter-monitoring.service'; export * from './lib/apis/asset-management.service'; +export * from './lib/apis/compact-pipeline.service'; export * from './lib/apis/data-view-data-explorer.service'; export * from './lib/apis/datalake-rest.service'; export * from './lib/apis/dashboard.service'; diff --git a/ui/src/app/connect/components/adapter-code-panel/adapter-code-panel.component.html b/ui/src/app/connect/components/adapter-code-panel/adapter-code-panel.component.html new file mode 100644 index 0000000000..8a05afabd3 --- /dev/null +++ b/ui/src/app/connect/components/adapter-code-panel/adapter-code-panel.component.html @@ -0,0 +1,25 @@ + + +
+ + +
diff --git a/ui/src/app/connect/components/adapter-code-panel/adapter-code-panel.component.scss b/ui/src/app/connect/components/adapter-code-panel/adapter-code-panel.component.scss new file mode 100644 index 0000000000..c02ba8acc6 --- /dev/null +++ b/ui/src/app/connect/components/adapter-code-panel/adapter-code-panel.component.scss @@ -0,0 +1,30 @@ +/*! + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +.preview-text { + background-color: black; + font: + 9pt Inconsolata, + monospace; + text-shadow: 0 0 5px #c8c8c8; + color: white; + padding: 10px; + max-width: 100%; + overflow-y: scroll; + white-space: pre-wrap; +} diff --git a/ui/src/app/connect/components/adapter-code-panel/adapter-code-panel.component.ts b/ui/src/app/connect/components/adapter-code-panel/adapter-code-panel.component.ts new file mode 100644 index 0000000000..5c742add6f --- /dev/null +++ b/ui/src/app/connect/components/adapter-code-panel/adapter-code-panel.component.ts @@ -0,0 +1,49 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +import { Component, Input, OnInit } from '@angular/core'; +import { + AdapterDescription, + AdapterService, + CompactAdapter, +} from '@streampipes/platform-services'; + +@Component({ + selector: 'sp-adapter-code-panel', + templateUrl: './adapter-code-panel.component.html', + styleUrls: ['./adapter-code-panel.component.scss'], +}) +export class AdapterCodePanelComponent implements OnInit { + @Input() + adapterDescription: AdapterDescription; + + @Input() + maxHeight = '300px'; + + compactAdapter: CompactAdapter; + + constructor(private adapterService: AdapterService) {} + + ngOnInit(): void { + this.adapterService + .convertToCompactAdapter(this.adapterDescription) + .subscribe(res => { + this.compactAdapter = res; + }); + } +} diff --git a/ui/src/app/connect/components/adapter-configuration/start-adapter-configuration/adapter-options-panel/adapter-options-panel.component.html b/ui/src/app/connect/components/adapter-configuration/start-adapter-configuration/adapter-options-panel/adapter-options-panel.component.html index 2bcfd840d1..b3afa5c944 100644 --- a/ui/src/app/connect/components/adapter-configuration/start-adapter-configuration/adapter-options-panel/adapter-options-panel.component.html +++ b/ui/src/app/connect/components/adapter-configuration/start-adapter-configuration/adapter-options-panel/adapter-options-panel.component.html @@ -18,17 +18,16 @@
-
- {{ optionIcon }} -
-
+
+ {{ optionIcon }}
-
+
{{ optionTitle }}
- {{ - optionDescription - }} + {{ optionDescription }} + +
- -
diff --git a/ui/src/app/connect/components/adapter-configuration/start-adapter-configuration/start-adapter-configuration.component.html b/ui/src/app/connect/components/adapter-configuration/start-adapter-configuration/start-adapter-configuration.component.html index 66cbbed55f..f7ff5b8c7b 100644 --- a/ui/src/app/connect/components/adapter-configuration/start-adapter-configuration/start-adapter-configuration.component.html +++ b/ui/src/app/connect/components/adapter-configuration/start-adapter-configuration/start-adapter-configuration.component.html @@ -145,6 +145,21 @@ + + + @if (showCode) { + + + } +
diff --git a/ui/src/app/connect/components/adapter-configuration/start-adapter-configuration/start-adapter-configuration.component.ts b/ui/src/app/connect/components/adapter-configuration/start-adapter-configuration/start-adapter-configuration.component.ts index e6b97f915a..e8fc266424 100644 --- a/ui/src/app/connect/components/adapter-configuration/start-adapter-configuration/start-adapter-configuration.component.ts +++ b/ui/src/app/connect/components/adapter-configuration/start-adapter-configuration/start-adapter-configuration.component.ts @@ -89,6 +89,7 @@ export class StartAdapterConfigurationComponent implements OnInit { dataLakeTimestampField: string; startAdapterNow = true; + showCode = false; constructor( private dialogService: DialogService, diff --git a/ui/src/app/connect/components/adapter-details/abstract-adapter-details.directive.ts b/ui/src/app/connect/components/adapter-details/abstract-adapter-details.directive.ts index 394274eb12..1298c1dea9 100644 --- a/ui/src/app/connect/components/adapter-details/abstract-adapter-details.directive.ts +++ b/ui/src/app/connect/components/adapter-details/abstract-adapter-details.directive.ts @@ -28,7 +28,9 @@ import { SpBreadcrumbService, } from '@streampipes/shared-ui'; import { SpAdapterDetailsTabs } from './adapter-details-tabs'; +import { Directive } from '@angular/core'; +@Directive() export abstract class SpAbstractAdapterDetailsDirective { currentAdapterId: string; tabs: SpNavigationItem[] = []; diff --git a/ui/src/app/connect/components/adapter-details/adapter-details-code/adapter-details-code.component.html b/ui/src/app/connect/components/adapter-details/adapter-details-code/adapter-details-code.component.html new file mode 100644 index 0000000000..6a93d34e7f --- /dev/null +++ b/ui/src/app/connect/components/adapter-details/adapter-details-code/adapter-details-code.component.html @@ -0,0 +1,38 @@ + + + +
+ @if (adapter) { +
+ + +
+ } +
+ diff --git a/ui/src/app/connect/components/adapter-details/adapter-details-code/adapter-details-code.component.ts b/ui/src/app/connect/components/adapter-details/adapter-details-code/adapter-details-code.component.ts new file mode 100644 index 0000000000..df2654b8d3 --- /dev/null +++ b/ui/src/app/connect/components/adapter-details/adapter-details-code/adapter-details-code.component.ts @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +import { Component, OnInit } from '@angular/core'; +import { SpAbstractAdapterDetailsDirective } from '../abstract-adapter-details.directive'; + +@Component({ + selector: 'sp-adapter-details-code', + templateUrl: './adapter-details-code.component.html', +}) +export class AdapterDetailsCodeComponent + extends SpAbstractAdapterDetailsDirective + implements OnInit +{ + ngOnInit() { + super.onInit(); + } + + onAdapterLoaded(): void {} +} diff --git a/ui/src/app/connect/components/adapter-details/adapter-details-tabs.ts b/ui/src/app/connect/components/adapter-details/adapter-details-tabs.ts index c4633a0fb4..8d3bf34714 100644 --- a/ui/src/app/connect/components/adapter-details/adapter-details-tabs.ts +++ b/ui/src/app/connect/components/adapter-details/adapter-details-tabs.ts @@ -36,6 +36,11 @@ export class SpAdapterDetailsTabs { itemTitle: 'Logs', itemLink: ['connect', 'details', elementId, 'logs'], }, + { + itemId: 'code', + itemTitle: 'Code', + itemLink: ['connect', 'details', elementId, 'code'], + }, ]; } } diff --git a/ui/src/app/connect/connect.module.ts b/ui/src/app/connect/connect.module.ts index f66f7edebb..33e5fce983 100644 --- a/ui/src/app/connect/connect.module.ts +++ b/ui/src/app/connect/connect.module.ts @@ -62,7 +62,6 @@ import { EditValueTransformationComponent } from './dialog/edit-event-property/c import { SpEpSettingsSectionComponent } from './dialog/edit-event-property/components/ep-settings-section/ep-settings-section.component'; import { SpAdapterOptionsPanelComponent } from './components/adapter-configuration/start-adapter-configuration/adapter-options-panel/adapter-options-panel.component'; import { SpAdapterTemplateDialogComponent } from './dialog/adapter-template/adapter-template-dialog.component'; -import { JsonPrettyPrintPipe } from './filter/json-pretty-print.pipe'; import { MatSnackBarModule } from '@angular/material/snack-bar'; import { AdapterConfigurationHeaderComponent } from './components/adapter-configuration/adapter-configuration-header/adapter-configuration-header.component'; import { NewAdapterComponent } from './components/new-adapter/new-adapter.component'; @@ -105,6 +104,8 @@ import { SpAdapterDeploymentSettingsComponent } from './components/adapter-confi import { SpAdapterDocumentationDialogComponent } from './dialog/adapter-documentation/adapter-documentation-dialog.component'; import { AdapterDetailsDataComponent } from './components/adapter-details/adapter-details-data/adapter-details-data.component'; import { EditRegexTransformationComponent } from './dialog/edit-event-property/components/edit-regex-transformation/edit-regex-transformation.component'; +import { AdapterCodePanelComponent } from './components/adapter-code-panel/adapter-code-panel.component'; +import { AdapterDetailsCodeComponent } from './components/adapter-details/adapter-details-code/adapter-details-code.component'; @NgModule({ imports: [ @@ -185,6 +186,10 @@ import { EditRegexTransformationComponent } from './dialog/edit-event-property/c path: 'logs', component: SpAdapterDetailsLogsComponent, }, + { + path: 'code', + component: AdapterDetailsCodeComponent, + }, ], }, ], @@ -194,9 +199,11 @@ import { EditRegexTransformationComponent } from './dialog/edit-event-property/c ], exports: [ErrorMessageComponent], declarations: [ + AdapterCodePanelComponent, AdapterConfigurationHeaderComponent, AdapterConfigurationComponent, AdapterDescriptionComponent, + AdapterDetailsCodeComponent, AdapterDetailsDataComponent, AdapterStartedDialog, AdapterStatusLightComponent, @@ -215,7 +222,6 @@ import { EditRegexTransformationComponent } from './dialog/edit-event-property/c EventSchemaPreviewComponent, ExistingAdaptersComponent, AdapterFilterPipe, - JsonPrettyPrintPipe, AdapterConfigurationComponent, TimestampPipe, EditCorrectionValueComponent, diff --git a/ui/src/app/connect/dialog/adapter-started/adapter-started-dialog.component.ts b/ui/src/app/connect/dialog/adapter-started/adapter-started-dialog.component.ts index 9d385b4516..598dae37ae 100644 --- a/ui/src/app/connect/dialog/adapter-started/adapter-started-dialog.component.ts +++ b/ui/src/app/connect/dialog/adapter-started/adapter-started-dialog.component.ts @@ -21,6 +21,8 @@ import { ShepherdService } from '../../../services/tour/shepherd.service'; import { AdapterDescription, AdapterService, + CompactPipeline, + CompactPipelineElement, ErrorMessage, PipelineOperationStatus, PipelineTemplateService, @@ -28,7 +30,7 @@ import { SpLogMessage, } from '@streampipes/platform-services'; import { DialogRef } from '@streampipes/shared-ui'; -import { PipelineInvocationBuilder } from '../../../core-services/template/PipelineInvocationBuilder'; +import { CompactPipelineService } from '../../../../../projects/streampipes/platform-services/src/lib/apis/compact-pipeline.service'; @Component({ selector: 'sp-dialog-adapter-started-dialog', @@ -80,6 +82,7 @@ export class AdapterStartedDialog implements OnInit { private adapterService: AdapterService, private shepherdService: ShepherdService, private pipelineTemplateService: PipelineTemplateService, + private compactPipelineService: CompactPipelineService, ) {} ngOnInit() { @@ -201,57 +204,68 @@ export class AdapterStartedDialog implements OnInit { private startSaveInDataLakePipeline(adapterElementId: string) { this.loadingText = 'Creating pipeline to persist data stream'; this.adapterService.getAdapter(adapterElementId).subscribe(adapter => { - const pipelineId = - 'org.apache.streampipes.manager.template.instances.DataLakePipelineTemplate'; this.pipelineTemplateService - .getPipelineTemplateInvocation( - adapter.correspondingDataStreamElementId, - pipelineId, - ) + .findById('sp-internal-persist') .subscribe( - res => { - const pipelineName = 'Persist ' + this.adapter.name; - - const indexName = this.adapter.name; - - const pipelineInvocation = - PipelineInvocationBuilder.create(res) - .setName(pipelineName) - .setTemplateId(pipelineId) - .setFreeTextStaticProperty( - 'db_measurement', - indexName, - ) - .setMappingPropertyUnary( - 'timestamp_mapping', - 's0::' + this.dataLakeTimestampField, - ) - .setOneOfStaticProperty( - 'schema_update', - 'Update schema', - ) - .build(); - - this.pipelineTemplateService - .createPipelineTemplateInvocation( - pipelineInvocation, - ) - .subscribe( - pipelineOperationStatus => { - this.pipelineOperationStatus = - pipelineOperationStatus; - this.startAdapter(adapterElementId, true); - }, - error => { - this.onAdapterFailure(error.error); - }, - ); + template => { + const pipeline: CompactPipeline = { + id: + 'persist-' + + this.adapter.name.replaceAll(' ', '-'), + name: 'Persist ' + this.adapter.name, + description: '', + pipelineElements: this.makeTemplateConfigs( + template.pipeline, + adapter, + ), + createOptions: { + persist: false, + start: true, + }, + }; + this.compactPipelineService.create(pipeline).subscribe( + pipelineOperationStatus => { + this.pipelineOperationStatus = + pipelineOperationStatus; + this.startAdapter(adapterElementId, true); + }, + error => { + this.onAdapterFailure(error.error); + }, + ); }, - res => { - this.templateErrorMessage = res.error; + error => { + this.templateErrorMessage = error.error; this.startAdapter(adapterElementId); }, ); }); } + + makeTemplateConfigs( + template: CompactPipelineElement[], + adapter: AdapterDescription, + ): CompactPipelineElement[] { + template[0].configuration.push( + { + db_measurement: this.adapter.name, + }, + { + timestamp_mapping: 's0::' + this.dataLakeTimestampField, + }, + { + dimensions_selection: adapter.eventSchema.eventProperties + .filter(ep => ep.propertyScope === 'DIMENSION_PROPERTY') + .map(ep => ep.runtimeName), + }, + ); + template.push({ + type: 'stream', + ref: 'stream1', + configuration: undefined, + id: adapter.correspondingDataStreamElementId, + connectedTo: undefined, + }); + return template; + } } diff --git a/ui/src/app/core-ui/configuration-code-panel/configuration-code-panel.component.html b/ui/src/app/core-ui/configuration-code-panel/configuration-code-panel.component.html new file mode 100644 index 0000000000..083d9291c7 --- /dev/null +++ b/ui/src/app/core-ui/configuration-code-panel/configuration-code-panel.component.html @@ -0,0 +1,34 @@ + + + + + 

+    
+    
+        

+    
+
diff --git a/ui/src/app/core-ui/configuration-code-panel/configuration-code-panel.component.scss b/ui/src/app/core-ui/configuration-code-panel/configuration-code-panel.component.scss
new file mode 100644
index 0000000000..c02ba8acc6
--- /dev/null
+++ b/ui/src/app/core-ui/configuration-code-panel/configuration-code-panel.component.scss
@@ -0,0 +1,30 @@
+/*!
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+.preview-text {
+    background-color: black;
+    font:
+        9pt Inconsolata,
+        monospace;
+    text-shadow: 0 0 5px #c8c8c8;
+    color: white;
+    padding: 10px;
+    max-width: 100%;
+    overflow-y: scroll;
+    white-space: pre-wrap;
+}
diff --git a/ui/src/app/core-ui/configuration-code-panel/configuration-code-panel.component.ts b/ui/src/app/core-ui/configuration-code-panel/configuration-code-panel.component.ts
new file mode 100644
index 0000000000..bf8deb6418
--- /dev/null
+++ b/ui/src/app/core-ui/configuration-code-panel/configuration-code-panel.component.ts
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+import { Component, Input, OnInit } from '@angular/core';
+
+@Component({
+    selector: 'sp-configuration-code-panel',
+    templateUrl: './configuration-code-panel.component.html',
+    styleUrls: ['./configuration-code-panel.component.scss'],
+})
+export class ConfigurationCodePanelComponent {
+    @Input()
+    configuration: any;
+
+    @Input()
+    maxHeight = '300px';
+}
diff --git a/ui/src/app/core-ui/core-ui.module.ts b/ui/src/app/core-ui/core-ui.module.ts
index dc2337b5d1..5a8815dce0 100644
--- a/ui/src/app/core-ui/core-ui.module.ts
+++ b/ui/src/app/core-ui/core-ui.module.ts
@@ -116,6 +116,9 @@ import { SingleMarkerMapComponent } from './single-marker-map/single-marker-map.
 import { LeafletModule } from '@asymmetrik/ngx-leaflet';
 import { StaticTreeInputTextEditorComponent } from './static-properties/static-runtime-resolvable-tree-input/static-tree-input-text-editor/static-tree-input-text-editor.component';
 import { PipelineElementTemplateConfigItemComponent } from './pipeline-element-template-config/pipeline-element-template-config-item/pipeline-element-template-config-item.component';
+import { ConfigurationCodePanelComponent } from './configuration-code-panel/configuration-code-panel.component';
+import { JsonPrettyPrintPipe } from './pipes/json-pretty-print.pipe';
+import { YamlPrettyPrintPipe } from './pipes/yaml-pretty-print.pipe';
 
 @NgModule({
     imports: [
@@ -168,6 +171,7 @@ import { PipelineElementTemplateConfigItemComponent } from './pipeline-element-t
         LeafletModule,
     ],
     declarations: [
+        ConfigurationCodePanelComponent,
         DataDownloadDialogComponent,
         DateInputComponent,
         DisplayRecommendedPipe,
@@ -220,9 +224,12 @@ import { PipelineElementTemplateConfigItemComponent } from './pipeline-element-t
         StatusIndicatorComponent,
         MultiStepStatusIndicatorComponent,
         PipelineOperationStatusComponent,
+        JsonPrettyPrintPipe,
+        YamlPrettyPrintPipe,
     ],
     providers: [MatDatepickerModule, DisplayRecommendedPipe],
     exports: [
+        ConfigurationCodePanelComponent,
         DataDownloadDialogComponent,
         DateInputComponent,
         PipelineElementTemplateConfigComponent,
@@ -255,6 +262,8 @@ import { PipelineElementTemplateConfigItemComponent } from './pipeline-element-t
         MultiStepStatusIndicatorComponent,
         PipelineOperationStatusComponent,
         SingleMarkerMapComponent,
+        JsonPrettyPrintPipe,
+        YamlPrettyPrintPipe,
     ],
 })
 export class CoreUiModule {}
diff --git a/ui/src/app/connect/filter/json-pretty-print.pipe.ts b/ui/src/app/core-ui/pipes/json-pretty-print.pipe.ts
similarity index 100%
rename from ui/src/app/connect/filter/json-pretty-print.pipe.ts
rename to ui/src/app/core-ui/pipes/json-pretty-print.pipe.ts
diff --git a/ui/src/app/core-ui/pipes/yaml-pretty-print.pipe.ts b/ui/src/app/core-ui/pipes/yaml-pretty-print.pipe.ts
new file mode 100644
index 0000000000..88f8b4b459
--- /dev/null
+++ b/ui/src/app/core-ui/pipes/yaml-pretty-print.pipe.ts
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+import { Injectable, Pipe, PipeTransform } from '@angular/core';
+import { parse, stringify } from 'yaml';
+
+@Pipe({
+    name: 'yamlpretty',
+})
+@Injectable({ providedIn: 'root' })
+export class YamlPrettyPrintPipe implements PipeTransform {
+    transform(obj: any) {
+        return stringify(obj).replace(/ /g, ' ').replace(/\n/g, '
');
+    }
+}
diff --git a/ui/src/app/editor/components/pipeline-assembly/pipeline-assembly-options/pipeline-assembly-options.component.html b/ui/src/app/editor/components/pipeline-assembly/pipeline-assembly-options/pipeline-assembly-options.component.html
index bbef21c3be..f15c990c06 100644
--- a/ui/src/app/editor/components/pipeline-assembly/pipeline-assembly-options/pipeline-assembly-options.component.html
+++ b/ui/src/app/editor/components/pipeline-assembly/pipeline-assembly-options/pipeline-assembly-options.component.html
@@ -50,23 +50,37 @@
     
     
     
+    
     
+    
      = new EventEmitter();
 
+    @Output()
+    displayPipelineTemplateEmitter: EventEmitter =
+        new EventEmitter();
+
     @ViewChild('assemblyOptionsPipelineCacheComponent')
     assemblyOptionsCacheComponent: PipelineAssemblyOptionsPipelineCacheComponent;
 
@@ -104,6 +112,21 @@ export class PipelineAssemblyOptionsComponent {
         });
     }
 
+    openAddTemplateDialog() {
+        const dialogRef = this.dialogService.open(AddTemplateDialogComponent, {
+            panelType: PanelType.SLIDE_IN_PANEL,
+            title: 'Add template',
+            width: '50vw',
+            data: {},
+        });
+        dialogRef.afterClosed().subscribe(pipeline => {
+            if (pipeline !== undefined) {
+                this.clearAssemblyEmitter.emit();
+                this.displayPipelineTemplateEmitter.emit(pipeline);
+            }
+        });
+    }
+
     showClearAssemblyConfirmDialog(event: any) {
         const dialogRef = this.dialog.open(ConfirmDialogComponent, {
             width: '500px',
diff --git a/ui/src/app/editor/components/pipeline-assembly/pipeline-assembly.component.html b/ui/src/app/editor/components/pipeline-assembly/pipeline-assembly.component.html
index e7d49b22b8..249eb2b489 100644
--- a/ui/src/app/editor/components/pipeline-assembly/pipeline-assembly.component.html
+++ b/ui/src/app/editor/components/pipeline-assembly/pipeline-assembly.component.html
@@ -28,11 +28,13 @@
             (clearAssemblyEmitter)="clearAssembly()"
             (togglePreviewEmitter)="togglePreview()"
             (savePipelineEmitter)="submit()"
+            (displayPipelineTemplateEmitter)="displayPipelineTemplate($event)"
         >
         
     

     

          {
+            this.rawPipelineModel = this.jsplumbService.makeRawPipeline(
+                pipeline,
+                false,
+            );
+            this.drawingAreaComponent.displayPipelineInEditor(true, undefined);
+        });
+    }
 }
diff --git a/ui/src/app/editor/dialog/add-template-dialog/add-template-dialog.component.html b/ui/src/app/editor/dialog/add-template-dialog/add-template-dialog.component.html
new file mode 100644
index 0000000000..981ebee58d
--- /dev/null
+++ b/ui/src/app/editor/dialog/add-template-dialog/add-template-dialog.component.html
@@ -0,0 +1,41 @@
+
+
+

+    

+        

+            
+            
+        

+    

+    
+    

+        
+        
+    

+

diff --git a/ui/src/app/editor/dialog/add-template-dialog/add-template-dialog.component.ts b/ui/src/app/editor/dialog/add-template-dialog/add-template-dialog.component.ts
new file mode 100644
index 0000000000..9ea500ffe8
--- /dev/null
+++ b/ui/src/app/editor/dialog/add-template-dialog/add-template-dialog.component.ts
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+import { Component, OnInit } from '@angular/core';
+import {
+    CompactPipelineTemplate,
+    PipelineTemplateGenerationRequest,
+    PipelineTemplateService,
+} from '@streampipes/platform-services';
+import { DialogRef } from '@streampipes/shared-ui';
+
+@Component({
+    selector: 'sp-add-template-dialog',
+    templateUrl: './add-template-dialog.component.html',
+})
+export class AddTemplateDialogComponent implements OnInit {
+    pipelineTemplates: CompactPipelineTemplate[] = [];
+
+    constructor(
+        private pipelineTemplateService: PipelineTemplateService,
+        private dialogRef: DialogRef,
+    ) {}
+
+    ngOnInit() {
+        this.pipelineTemplateService
+            .findAll()
+            .subscribe(t => (this.pipelineTemplates = t));
+    }
+
+    selectTemplate(template: CompactPipelineTemplate): void {
+        const req: PipelineTemplateGenerationRequest = {
+            streams: {},
+            template,
+            pipelineName: undefined,
+            pipelineDescription: undefined,
+        };
+        this.pipelineTemplateService
+            .getPipelineForTemplate(template.elementId, req)
+            .subscribe(pipeline => {
+                this.dialogRef.close(pipeline);
+            });
+    }
+
+    close(): void {
+        this.dialogRef.close();
+    }
+}
diff --git a/ui/src/app/editor/dialog/add-template-dialog/template-selection/template-selection.component.html b/ui/src/app/editor/dialog/add-template-dialog/template-selection/template-selection.component.html
new file mode 100644
index 0000000000..58f92cc707
--- /dev/null
+++ b/ui/src/app/editor/dialog/add-template-dialog/template-selection/template-selection.component.html
@@ -0,0 +1,39 @@
+
+
+

+    @for (template of pipelineTemplates; track template.elementId) {
+        

+            

+                

+                    {{ template.name }}
+                

+                

+                    {{ template.description }}
+                

+            

+        

+        
+    }
+

diff --git a/ui/src/app/editor/dialog/add-template-dialog/template-selection/template-selection.component.scss b/ui/src/app/editor/dialog/add-template-dialog/template-selection/template-selection.component.scss
new file mode 100644
index 0000000000..0502d36fa9
--- /dev/null
+++ b/ui/src/app/editor/dialog/add-template-dialog/template-selection/template-selection.component.scss
@@ -0,0 +1,26 @@
+/*!
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+.template-item:nth-child(even) {
+    background: var(--color-bg-1);
+}
+
+.template-item:hover {
+    background: var(--color-bg-2);
+    cursor: pointer;
+}
diff --git a/ui/src/app/editor/dialog/add-template-dialog/template-selection/template-selection.component.ts b/ui/src/app/editor/dialog/add-template-dialog/template-selection/template-selection.component.ts
new file mode 100644
index 0000000000..3cbd628ed1
--- /dev/null
+++ b/ui/src/app/editor/dialog/add-template-dialog/template-selection/template-selection.component.ts
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+import { Component, EventEmitter, Input, Output } from '@angular/core';
+import { CompactPipelineTemplate } from '@streampipes/platform-services';
+
+@Component({
+    selector: 'sp-template-selection',
+    templateUrl: './template-selection.component.html',
+    styleUrls: ['./template-selection.component.scss'],
+})
+export class TemplateSelectionComponent {
+    @Input()
+    pipelineTemplates: CompactPipelineTemplate[] = [];
+
+    @Output()
+    selectTemplateEmitter: EventEmitter =
+        new EventEmitter();
+
+    selectTemplate(template: CompactPipelineTemplate): void {
+        this.selectTemplateEmitter.emit(template);
+    }
+}
diff --git a/ui/src/app/editor/dialog/save-pipeline/save-pipeline-settings/save-pipeline-settings.component.html b/ui/src/app/editor/dialog/save-pipeline/save-pipeline-settings/save-pipeline-settings.component.html
index 6ff8ea6a14..18f221cbfc 100644
--- a/ui/src/app/editor/dialog/save-pipeline/save-pipeline-settings/save-pipeline-settings.component.html
+++ b/ui/src/app/editor/dialog/save-pipeline/save-pipeline-settings/save-pipeline-settings.component.html
@@ -63,17 +63,15 @@
                     (blur)="triggerTutorial()"
                 />
                 Pipeline name must have between 3 and 40
-                    characters.
+                    >Pipeline name must have between 3 and 40 characters.
+                
             
             
                 Description
                 
                 Pipeline description must not have more than 80
-                    characters.
+                    >Pipeline description must not have more than 80 characters.
+                
             
         

     
@@ -91,4 +89,16 @@
     >
         Navigate to pipeline overview afterwards
     
+    

+        
+            Show pipeline configuration as code
+            
+            
+        
+    

 

diff --git a/ui/src/app/editor/dialog/save-pipeline/save-pipeline-settings/save-pipeline-settings.component.scss b/ui/src/app/editor/dialog/save-pipeline/save-pipeline-settings/save-pipeline-settings.component.scss
index 13cbc4aacb..e22ed79985 100644
--- a/ui/src/app/editor/dialog/save-pipeline/save-pipeline-settings/save-pipeline-settings.component.scss
+++ b/ui/src/app/editor/dialog/save-pipeline/save-pipeline-settings/save-pipeline-settings.component.scss
@@ -15,3 +15,7 @@
  * limitations under the License.
  *
  */
+
+.border-1 {
+    border: 1px solid var(--color-bg-2);
+}
diff --git a/ui/src/app/editor/dialog/save-pipeline/save-pipeline-settings/save-pipeline-settings.component.ts b/ui/src/app/editor/dialog/save-pipeline/save-pipeline-settings/save-pipeline-settings.component.ts
index d75b2c1aed..fb0daa47bb 100644
--- a/ui/src/app/editor/dialog/save-pipeline/save-pipeline-settings/save-pipeline-settings.component.ts
+++ b/ui/src/app/editor/dialog/save-pipeline/save-pipeline-settings/save-pipeline-settings.component.ts
@@ -23,7 +23,11 @@ import {
     UntypedFormGroup,
     Validators,
 } from '@angular/forms';
-import { Pipeline } from '@streampipes/platform-services';
+import {
+    CompactPipeline,
+    Pipeline,
+    PipelineService,
+} from '@streampipes/platform-services';
 import { PipelineStorageOptions } from '../../../model/editor.model';
 
 @Component({
@@ -44,7 +48,12 @@ export class SavePipelineSettingsComponent implements OnInit {
     @Input()
     currentPipelineName: string;
 
-    constructor(private shepherdService: ShepherdService) {}
+    compactPipeline: CompactPipeline;
+
+    constructor(
+        private shepherdService: ShepherdService,
+        private pipelineService: PipelineService,
+    ) {}
 
     ngOnInit() {
         this.submitPipelineForm.addControl(
@@ -73,6 +82,9 @@ export class SavePipelineSettingsComponent implements OnInit {
         ].valueChanges.subscribe(value => {
             this.pipeline.description = value;
         });
+        this.pipelineService
+            .convertToCompactPipeline(this.pipeline)
+            .subscribe(p => (this.compactPipeline = p));
     }
 
     triggerTutorial() {
diff --git a/ui/src/app/editor/editor.module.ts b/ui/src/app/editor/editor.module.ts
index 0995cda75a..c2742c7199 100644
--- a/ui/src/app/editor/editor.module.ts
+++ b/ui/src/app/editor/editor.module.ts
@@ -84,6 +84,8 @@ import { DroppedPipelineElementComponent } from './components/pipeline/dropped-p
 import { InputSchemaPanelComponent } from './dialog/customize/input-schema-panel/input-schema-panel.component';
 import { InputSchemaPropertyComponent } from './dialog/customize/input-schema-panel/input-schema-property/input-schema-property.component';
 import { SortByRuntimeNamePipe } from './pipes/sort-by-runtime-name.pipe';
+import { AddTemplateDialogComponent } from './dialog/add-template-dialog/add-template-dialog.component';
+import { TemplateSelectionComponent } from './dialog/add-template-dialog/template-selection/template-selection.component';
 
 @NgModule({
     imports: [
@@ -125,6 +127,7 @@ import { SortByRuntimeNamePipe } from './pipes/sort-by-runtime-name.pipe';
         PlatformServicesModule,
     ],
     declarations: [
+        AddTemplateDialogComponent,
         CompatibleElementsComponent,
         CustomizeComponent,
         CustomOutputStrategyComponent,
@@ -155,6 +158,7 @@ import { SortByRuntimeNamePipe } from './pipes/sort-by-runtime-name.pipe';
         PipelineElementTypeFilterPipe,
         PipelineComponent,
         PropertySelectionComponent,
+        TemplateSelectionComponent,
         SavePipelineComponent,
         SavePipelineSettingsComponent,
         SortByRuntimeNamePipe,
diff --git a/ui/src/app/editor/services/pipeline-positioning.service.ts b/ui/src/app/editor/services/pipeline-positioning.service.ts
index 52342e9e66..853568d11f 100644
--- a/ui/src/app/editor/services/pipeline-positioning.service.ts
+++ b/ui/src/app/editor/services/pipeline-positioning.service.ts
@@ -165,6 +165,7 @@ export class PipelinePositioningService {
             elementRef.css('left', g.node(v).x + 'px');
             elementRef.css('top', g.node(v).y + 'px');
         });
+        jsPlumbBridge.repaintEverything();
     }
 
     layoutGraphFromCanvasMetadata(
diff --git a/ui/src/app/pipeline-details/components/pipeline-details-toolbar/pipeline-details-toolbar.component.html b/ui/src/app/pipeline-details/components/pipeline-details-toolbar/pipeline-details-toolbar.component.html
index 4a6192901c..01e230ab4a 100644
--- a/ui/src/app/pipeline-details/components/pipeline-details-toolbar/pipeline-details-toolbar.component.html
+++ b/ui/src/app/pipeline-details/components/pipeline-details-toolbar/pipeline-details-toolbar.component.html
@@ -30,6 +30,18 @@
              Disable live preview
         
     
+    
     

     

         
+    

+
diff --git a/ui/src/app/pipeline-details/dialogs/pipeline-code/pipeline-code-dialog.component.ts b/ui/src/app/pipeline-details/dialogs/pipeline-code/pipeline-code-dialog.component.ts
new file mode 100644
index 0000000000..f7c96f031e
--- /dev/null
+++ b/ui/src/app/pipeline-details/dialogs/pipeline-code/pipeline-code-dialog.component.ts
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+import { Component, Input, OnInit } from '@angular/core';
+import {
+    CompactPipeline,
+    Pipeline,
+    PipelineService,
+} from '@streampipes/platform-services';
+import { DialogRef } from '@streampipes/shared-ui';
+
+@Component({
+    selector: 'sp-pipeline-code-dialog',
+    templateUrl: './pipeline-code-dialog.component.html',
+})
+export class PipelineCodeDialogComponent implements OnInit {
+    @Input()
+    pipeline: Pipeline;
+
+    compactPipeline: CompactPipeline;
+
+    constructor(
+        private pipelineService: PipelineService,
+        private dialogRef: DialogRef,
+    ) {}
+
+    ngOnInit() {
+        this.pipelineService
+            .convertToCompactPipeline(this.pipeline)
+            .subscribe(p => (this.compactPipeline = p));
+    }
+
+    close() {
+        this.dialogRef.close();
+    }
+}
diff --git a/ui/src/app/pipeline-details/pipeline-details.component.html b/ui/src/app/pipeline-details/pipeline-details.component.html
index 500885bccd..e012942490 100644
--- a/ui/src/app/pipeline-details/pipeline-details.component.html
+++ b/ui/src/app/pipeline-details/pipeline-details.component.html
@@ -29,6 +29,7 @@
             (togglePreviewEmitter)="toggleLivePreview()"
             (autoRefreshChange)="autoRefresh = $event"
             (reloadMetricsEmitter)="triggerReload()"
+            (openCodeDialogEmitter)="openPipelineAsCodeDialog()"
         >
         
     
diff --git a/ui/src/app/pipeline-details/pipeline-details.component.ts b/ui/src/app/pipeline-details/pipeline-details.component.ts
index 97c070b1b9..19ec1c45ac 100644
--- a/ui/src/app/pipeline-details/pipeline-details.component.ts
+++ b/ui/src/app/pipeline-details/pipeline-details.component.ts
@@ -31,6 +31,8 @@ import {
 import { PipelineElementUnion } from '../editor/model/editor.model';
 import {
     CurrentUserService,
+    DialogService,
+    PanelType,
     SpBreadcrumbService,
 } from '@streampipes/shared-ui';
 import { SpPipelineRoutes } from '../pipelines/pipelines.routes';
@@ -40,6 +42,7 @@ import { catchError, filter, switchMap } from 'rxjs/operators';
 import { PipelinePreviewComponent } from './components/preview/pipeline-preview.component';
 import { HttpContext } from '@angular/common/http';
 import { NGX_LOADING_BAR_IGNORED } from '@ngx-loading-bar/http-client';
+import { PipelineCodeDialogComponent } from './dialogs/pipeline-code/pipeline-code-dialog.component';
 
 @Component({
     selector: 'sp-pipeline-details-overview-component',
@@ -75,6 +78,7 @@ export class SpPipelineDetailsComponent implements OnInit, OnDestroy {
         private currentUserService: CurrentUserService,
         private breadcrumbService: SpBreadcrumbService,
         private pipelineMonitoringService: PipelineMonitoringService,
+        private dialogService: DialogService,
     ) {}
 
     ngOnInit(): void {
@@ -176,6 +180,17 @@ export class SpPipelineDetailsComponent implements OnInit, OnDestroy {
         this.pipelinePreviewComponent?.toggleLivePreview();
     }
 
+    openPipelineAsCodeDialog(): void {
+        this.dialogService.open(PipelineCodeDialogComponent, {
+            panelType: PanelType.SLIDE_IN_PANEL,
+            width: '50vw',
+            title: 'Pipeline code',
+            data: {
+                pipeline: this.pipeline,
+            },
+        });
+    }
+
     ngOnDestroy() {
         this.currentUserSub?.unsubscribe();
         this.autoRefreshSub?.unsubscribe();
diff --git a/ui/src/app/pipeline-details/pipeline-details.module.ts b/ui/src/app/pipeline-details/pipeline-details.module.ts
index 8b15a1fca6..f05009a578 100644
--- a/ui/src/app/pipeline-details/pipeline-details.module.ts
+++ b/ui/src/app/pipeline-details/pipeline-details.module.ts
@@ -41,6 +41,7 @@ import { MatTooltipModule } from '@angular/material/tooltip';
 import { PipelineDetailsToolbarComponent } from './components/pipeline-details-toolbar/pipeline-details-toolbar.component';
 import { MatSlideToggleModule } from '@angular/material/slide-toggle';
 import { MatDivider } from '@angular/material/divider';
+import { PipelineCodeDialogComponent } from './dialogs/pipeline-code/pipeline-code-dialog.component';
 
 @NgModule({
     imports: [
@@ -64,6 +65,7 @@ import { MatDivider } from '@angular/material/divider';
     ],
     declarations: [
         PipelineActionsComponent,
+        PipelineCodeDialogComponent,
         PipelineElementsRowComponent,
         PipelineLogsDialogComponent,
         PipelineStatusComponent,